Upload folder using huggingface_hub

README.md

@@ -0,0 +1,61 @@
+---
+tags:
+- merge
+- parameter_wise
+- llm-adamerge
+base_model: mistralai/Mistral-7B-v0.1
+---
+
+# Merged Model using LLM-AdaMerge (parameter_wise)
+
+This model was created by merging multiple fine-tuned models using the LLM-AdaMerge approach with parameter_wise merging.
+
+## Merge Details
+
+- **Merge Type**: parameter_wise
+- **Base Model**: mistralai/Mistral-7B-v0.1
+- **Number of Models Merged**: 3
+- **Models Merged**: instruct, math, code
+- **Final Training Loss**: N/A
+- **Training Epochs**: 0
+
+## Lambda Coefficients
+
+The following lambda coefficients were learned during training:
+
+
+### Parameter-wise Lambdas
+This model uses parameter-wise lambda coefficients. Total parameters with individual lambdas: 291
+
+
+See the uploaded `learned_lambdas.json` file for detailed parameter-wise coefficients.
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained("your-username/model-name")
+tokenizer = AutoTokenizer.from_pretrained("your-username/model-name")
+
+# Use the model
+inputs = tokenizer("Hello, how are you?", return_tensors="pt")
+outputs = model.generate(**inputs)
+print(tokenizer.decode(outputs[0]))
+```
+
+## Training Configuration
+
+See the uploaded `training_config.json` file for detailed training configuration.
+
+## Citation
+
+If you use this model, please cite the LLM-AdaMerge paper:
+
+```bibtex
+@article{llmadamerge2024,
+  title={LLM-AdaMerge: Adaptive Model Merging for Large Language Models},
+  author={...},
+  year={2024}
+}
+```

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": null,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.52.4"
+}

learned_lambdas.json

@@ -0,0 +1,1759 @@
+{
+  "lambdas": [
+    [
+      0.21359698474407196,
+      -0.2164446860551834,
+      0.4349040985107422
+    ],
+    [
+      0.18373654782772064,
+      0.07137170433998108,
+      0.06354912370443344
+    ],
+    [
+      0.24980150163173676,
+      -0.07857954502105713,
+      -0.3595312833786011
+    ],
+    [
+      0.7251716256141663,
+      -0.016075219959020615,
+      0.30653372406959534
+    ],
+    [
+      0.551445484161377,
+      -0.21878266334533691,
+      0.46370652318000793
+    ],
+    [
+      -0.27936112880706787,
+      0.46840620040893555,
+      -0.1450582891702652
+    ],
+    [
+      0.5147512555122375,
+      0.7473368048667908,
+      -0.570711612701416
+    ],
+    [
+      0.4900670647621155,
+      0.8384904861450195,
+      0.5350456833839417
+    ],
+    [
+      -0.3452461361885071,
+      -0.24068720638751984,
+      1.0286009311676025
+    ],
+    [
+      0.7639378309249878,
+      -0.3156850337982178,
+      -0.23452775180339813
+    ],
+    [
+      0.11127755045890808,
+      0.03986138477921486,
+      0.46316784620285034
+    ],
+    [
+      0.45491963624954224,
+      0.6616725921630859,
+      0.45508068799972534
+    ],
+    [
+      -0.2381044328212738,
+      -0.6323292255401611,
+      0.0649583488702774
+    ],
+    [
+      -0.4048399031162262,
+      0.620869517326355,
+      0.41412895917892456
+    ],
+    [
+      0.19767624139785767,
+      -0.3238150477409363,
+      -0.044296279549598694
+    ],
+    [
+      -0.21615006029605865,
+      0.16054067015647888,
+      0.03723854944109917
+    ],
+    [
+      0.6151573657989502,
+      0.5277937650680542,
+      0.7325867414474487
+    ],
+    [
+      0.256672203540802,
+      0.9549311995506287,
+      0.33641910552978516
+    ],
+    [
+      -0.07662014663219452,
+      -0.22492413222789764,
+      0.7069713473320007
+    ],
+    [
+      0.40927863121032715,
+      0.005207874812185764,
+      0.1688462346792221
+    ],
+    [
+      0.5177145004272461,
+      0.011327486485242844,
+      0.12756213545799255
+    ],
+    [
+      -0.3924155533313751,
+      0.2530467212200165,
+      -0.09741459041833878
+    ],
+    [
+      -0.3027581572532654,
+      0.3145124912261963,
+      0.4244309067726135
+    ],
+    [
+      0.7682188153266907,
+      0.36635830998420715,
+      0.666470468044281
+    ],
+    [
+      -0.09421560168266296,
+      0.5080885887145996,
+      -0.06889066100120544
+    ],
+    [
+      0.557958722114563,
+      0.4884584844112396,
+      0.6631458401679993
+    ],
+    [
+      0.017188655212521553,
+      0.5574567914009094,
+      -0.006177110597491264
+    ],
+    [
+      0.8893246054649353,
+      1.0660768747329712,
+      0.5864647030830383
+    ],
+    [
+      0.11193809658288956,
+      0.41452017426490784,
+      0.530289888381958
+    ],
+    [
+      0.007093043997883797,
+      0.5380092263221741,
+      0.5444429516792297
+    ],
+    [
+      0.8307387828826904,
+      0.12119932472705841,
+      0.5001442432403564
+    ],
+    [
+      -0.2514621615409851,
+      0.4150887131690979,
+      0.5546015501022339
+    ],
+    [
+      0.2250683754682541,
+      -0.3092842400074005,
+      0.4038548469543457
+    ],
+    [
+      -0.15786027908325195,
+      -0.576519250869751,
+      -0.18042252957820892
+    ],
+    [
+      -0.030206184834241867,
+      0.9736607670783997,
+      -0.0683356449007988
+    ],
+    [
+      -0.20779341459274292,
+      0.5420925617218018,
+      0.47638383507728577
+    ],
+    [
+      0.32098907232284546,
+      0.7035049796104431,
+      1.0407670736312866
+    ],
+    [
+      0.24926768243312836,
+      -0.03809899836778641,
+      0.3589838445186615
+    ],
+    [
+      0.2516117990016937,
+      0.0950184091925621,
+      0.12089581787586212
+    ],
+    [
+      0.16146092116832733,
+      0.5226585268974304,
+      -0.16740091145038605
+    ],
+    [
+      0.14300084114074707,
+      0.06114785745739937,
+      0.2410912662744522
+    ],
+    [
+      0.30880165100097656,
+      0.7231574058532715,
+      0.6492418646812439
+    ],
+    [
+      -0.39174404740333557,
+      0.391517698764801,
+      0.1175754964351654
+    ],
+    [
+      1.1349018812179565,
+      0.8942872285842896,
+      0.8756872415542603
+    ],
+    [
+      0.16054439544677734,
+      0.4537104666233063,
+      0.17235562205314636
+    ],
+    [
+      0.9249570965766907,
+      1.2538586854934692,
+      0.14110293984413147
+    ],
+    [
+      0.10978423058986664,
+      0.07031691074371338,
+      0.0028920783661305904
+    ],
+    [
+      -0.14549046754837036,
+      -0.1875956952571869,
+      -0.057581380009651184
+    ],
+    [
+      -0.13747359812259674,
+      0.45354631543159485,
+      0.69197016954422
+    ],
+    [
+      0.7033130526542664,
+      0.2677704691886902,
+      0.6307313442230225
+    ],
+    [
+      -0.026439161971211433,
+      0.5624685883522034,
+      0.5508381128311157
+    ],
+    [
+      0.5997487902641296,
+      0.6207756996154785,
+      0.3884807229042053
+    ],
+    [
+      0.9113128185272217,
+      0.5756756663322449,
+      0.5259497761726379
+    ],
+    [
+      -0.0764508843421936,
+      0.48238518834114075,
+      0.533565878868103
+    ],
+    [
+      0.3155931532382965,
+      0.44924691319465637,
+      -0.0018975046696141362
+    ],
+    [
+      0.06382375210523605,
+      -0.14388808608055115,
+      -0.23163174092769623
+    ],
+    [
+      0.5853031277656555,
+      0.32832714915275574,
+      0.054335832595825195
+    ],
+    [
+      -0.25589367747306824,
+      0.4742071330547333,
+      0.21511781215667725
+    ],
+    [
+      -0.752947986125946,
+      0.7910690903663635,
+      0.47580817341804504
+    ],
+    [
+      0.19347093999385834,
+      -0.45822176337242126,
+      0.3438996970653534
+    ],
+    [
+      0.4309948980808258,
+      0.3567085266113281,
+      0.6842737793922424
+    ],
+    [
+      0.7723644971847534,
+      0.26576516032218933,
+      0.8875527381896973
+    ],
+    [
+      0.1905166208744049,
+      -0.3240078389644623,
+      0.6336449980735779
+    ],
+    [
+      0.38887909054756165,
+      0.7544465065002441,
+      0.2487660050392151
+    ],
+    [
+      -0.08940726518630981,
+      -0.007276141084730625,
+      0.06268707662820816
+    ],
+    [
+      0.7145946621894836,
+      0.47930267453193665,
+      0.10245968401432037
+    ],
+    [
+      -0.36302587389945984,
+      -0.3691670596599579,
+      0.03408021479845047
+    ],
+    [
+      0.3267446756362915,
+      0.13149414956569672,
+      0.49767404794692993
+    ],
+    [
+      0.6062907576560974,
+      0.491150826215744,
+      0.7409757971763611
+    ],
+    [
+      0.45287612080574036,
+      -0.47713926434516907,
+      0.4198429584503174
+    ],
+    [
+      0.2924189567565918,
+      0.45667028427124023,
+      0.5962654948234558
+    ],
+    [
+      0.04065968468785286,
+      0.2760885953903198,
+      0.5384055376052856
+    ],
+    [
+      0.9806042909622192,
+      -0.018676593899726868,
+      -0.39374840259552
+    ],
+    [
+      0.49048981070518494,
+      -0.5551500916481018,
+      0.4733657240867615
+    ],
+    [
+      0.7472072243690491,
+      0.5198571681976318,
+      0.48487377166748047
+    ],
+    [
+      -0.14948683977127075,
+      0.17321796715259552,
+      -0.08548928052186966
+    ],
+    [
+      0.8185954093933105,
+      0.5525864362716675,
+      0.44340991973876953
+    ],
+    [
+      0.9454837441444397,
+      -0.00021949218353256583,
+      0.53125
+    ],
+    [
+      0.6961415410041809,
+      0.35851001739501953,
+      0.18167944252490997
+    ],
+    [
+      1.2112905979156494,
+      0.48523274064064026,
+      -0.44952115416526794
+    ],
+    [
+      0.6371596455574036,
+      0.2760885953903198,
+      -0.07913361489772797
+    ],
+    [
+      0.38398104906082153,
+      0.5510115027427673,
+      0.8780030012130737
+    ],
+    [
+      -0.4051967263221741,
+      -0.6793044805526733,
+      -0.4881371259689331
+    ],
+    [
+      -0.5237668752670288,
+      -0.17254097759723663,
+      -0.2170979380607605
+    ],
+    [
+      0.4532039761543274,
+      -0.8364969491958618,
+      0.9962347745895386
+    ],
+    [
+      -0.41449347138404846,
+      -0.5075752139091492,
+      0.704350471496582
+    ],
+    [
+      1.064165472984314,
+      0.19063466787338257,
+      0.5938963890075684
+    ],
+    [
+      0.4706924259662628,
+      0.9520235657691956,
+      0.1698094606399536
+    ],
+    [
+      0.5618039965629578,
+      0.05764932185411453,
+      0.2127338945865631
+    ],
+    [
+      -0.06680785864591599,
+      -0.21486449241638184,
+      -0.02948073484003544
+    ],
+    [
+      0.8544485569000244,
+      0.40998706221580505,
+      0.3041371703147888
+    ],
+    [
+      0.2568047046661377,
+      -0.046769797801971436,
+      -0.5896286368370056
+    ],
+    [
+      0.7117580771446228,
+      -0.6780204176902771,
+      -0.05612644925713539
+    ],
+    [
+      0.5117921233177185,
+      0.594724178314209,
+      0.015837913379073143
+    ],
+    [
+      0.11640041321516037,
+      0.3007548153400421,
+      -0.36630135774612427
+    ],
+    [
+      -0.2639734148979187,
+      -0.22972415387630463,
+      0.31596460938453674
+    ],
+    [
+      0.052716951817274094,
+      -0.5934274196624756,
+      0.6101188659667969
+    ],
+    [
+      0.10834594815969467,
+      0.7756256461143494,
+      0.15687476098537445
+    ],
+    [
+      -0.1295362114906311,
+      0.2760885953903198,
+      0.43015411496162415
+    ],
+    [
+      0.12494787573814392,
+      0.48390090465545654,
+      -0.1764613389968872
+    ],
+    [
+      -0.1907060146331787,
+      -0.2893322706222534,
+      -0.3830707371234894
+    ],
+    [
+      0.005892400164157152,
+      -0.3453017771244049,
+      -0.5945209860801697
+    ],
+    [
+      0.28648245334625244,
+      0.6291634440422058,
+      -0.03156203031539917
+    ],
+    [
+      -0.3965371549129486,
+      0.3238464593887329,
+      0.6217119693756104
+    ],
+    [
+      0.28561535477638245,
+      0.4140786826610565,
+      0.143829345703125
+    ],
+    [
+      0.37624838948249817,
+      0.4192754328250885,
+      0.04449336603283882
+    ],
+    [
+      0.9060468077659607,
+      0.8771811723709106,
+      0.2883681058883667
+    ],
+    [
+      -0.16548031568527222,
+      -0.306856244802475,
+      0.5969915986061096
+    ],
+    [
+      0.13628481328487396,
+      -0.373859703540802,
+      0.9392935633659363
+    ],
+    [
+      -0.23142965137958527,
+      -0.5544381737709045,
+      -0.24528397619724274
+    ],
+    [
+      0.7414894104003906,
+      -0.17250315845012665,
+      -0.0031167385168373585
+    ],
+    [
+      0.37525615096092224,
+      1.2675185203552246,
+      1.1016807556152344
+    ],
+    [
+      -0.729317307472229,
+      0.5185400247573853,
+      0.5889196395874023
+    ],
+    [
+      0.006818769033998251,
+      0.5573992729187012,
+      0.08504647016525269
+    ],
+    [
+      1.1147239208221436,
+      0.5571479201316833,
+      -0.035356659442186356
+    ],
+    [
+      0.7418893575668335,
+      0.2531592547893524,
+      0.562224268913269
+    ],
+    [
+      0.8112162351608276,
+      0.3641621768474579,
+      0.36308473348617554
+    ],
+    [
+      0.048483435064554214,
+      0.6834896802902222,
+      -0.6634699106216431
+    ],
+    [
+      0.22691766917705536,
+      -0.4956328272819519,
+      -1.0982667207717896
+    ],
+    [
+      0.6410916447639465,
+      -0.06540890783071518,
+      0.17114508152008057
+    ],
+    [
+      1.4179061651229858,
+      -0.4744257926940918,
+      0.6435526013374329
+    ],
+    [
+      -0.694256603717804,
+      0.5681807398796082,
+      -0.09950689226388931
+    ],
+    [
+      0.5402541756629944,
+      0.278556764125824,
+      0.43470650911331177
+    ],
+    [
+      -0.1546173393726349,
+      1.0311001539230347,
+      0.43332475423812866
+    ],
+    [
+      0.8175909519195557,
+      1.0657856464385986,
+      0.5252527594566345
+    ],
+    [
+      0.6673902273178101,
+      0.2760885953903198,
+      -0.14615967869758606
+    ],
+    [
+      0.6274813413619995,
+      -0.03086041286587715,
+      0.3763096332550049
+    ],
+    [
+      0.8328699469566345,
+      -0.37110206484794617,
+      0.4917232096195221
+    ],
+    [
+      0.7410853505134583,
+      1.0419585704803467,
+      1.1761726140975952
+    ],
+    [
+      0.34083282947540283,
+      0.9037180542945862,
+      0.6210451126098633
+    ],
+    [
+      -1.0043085813522339,
+      0.4448750615119934,
+      -0.23202694952487946
+    ],
+    [
+      0.5952733159065247,
+      0.3448507785797119,
+      0.68792724609375
+    ],
+    [
+      -0.14194385707378387,
+      0.1958417296409607,
+      0.07494542747735977
+    ],
+    [
+      -0.4235289692878723,
+      1.8409199714660645,
+      -0.5328864455223083
+    ],
+    [
+      0.6062782406806946,
+      0.3724144697189331,
+      -0.6195069551467896
+    ],
+    [
+      0.9738200902938843,
+      -0.19491901993751526,
+      -0.6321206092834473
+    ],
+    [
+      0.14497826993465424,
+      0.8698608875274658,
+      -0.19751141965389252
+    ],
+    [
+      0.35451802611351013,
+      -0.09179341793060303,
+      -0.2862659692764282
+    ],
+    [
+      0.20770026743412018,
+      0.4990694522857666,
+      0.597663402557373
+    ],
+    [
+      0.07840877026319504,
+      0.7478991150856018,
+      -0.2143201231956482
+    ],
+    [
+      -0.32848820090293884,
+      0.2380635142326355,
+      0.7736063003540039
+    ],
+    [
+      -0.24926325678825378,
+      -0.3298020362854004,
+      -0.6317358613014221
+    ],
+    [
+      -0.13794203102588654,
+      0.3890848755836487,
+      -0.09316663444042206
+    ],
+    [
+      0.05574212968349457,
+      0.2760885953903198,
+      0.543753445148468
+    ],
+    [
+      -0.1514638066291809,
+      0.23378300666809082,
+      0.6495851874351501
+    ],
+    [
+      0.7095686793327332,
+      0.12177275121212006,
+      -0.5433270931243896
+    ],
+    [
+      0.32450565695762634,
+      0.44656068086624146,
+      0.43729379773139954
+    ],
+    [
+      -0.59732985496521,
+      -0.13850414752960205,
+      -0.2800966203212738
+    ],
+    [
+      -0.14024443924427032,
+      1.3604012727737427,
+      -0.454708456993103
+    ],
+    [
+      0.3004085123538971,
+      -0.07340076565742493,
+      0.0004387743247207254
+    ],
+    [
+      0.15913432836532593,
+      0.6195625066757202,
+      -0.30247849225997925
+    ],
+    [
+      0.18473072350025177,
+      0.07634653896093369,
+      0.5509761571884155
+    ],
+    [
+      0.4589727818965912,
+      0.2760885953903198,
+      0.29281359910964966
+    ],
+    [
+      -0.20752620697021484,
+      -0.20893408358097076,
+      0.883582592010498
+    ],
+    [
+      0.3957076370716095,
+      0.4437480568885803,
+      1.0711257457733154
+    ],
+    [
+      1.0205185413360596,
+      0.8279246091842651,
+      1.1254446506500244
+    ],
+    [
+      0.488484263420105,
+      -0.6802319288253784,
+      0.064021535217762
+    ],
+    [
+      -0.5841817259788513,
+      0.003159634070470929,
+      0.3550851345062256
+    ],
+    [
+      -0.17998957633972168,
+      0.2949807941913605,
+      0.14250275492668152
+    ],
+    [
+      -0.21448534727096558,
+      0.40725982189178467,
+      0.4643062353134155
+    ],
+    [
+      0.5733755826950073,
+      0.7918974161148071,
+      0.8998477458953857
+    ],
+    [
+      0.8696557283401489,
+      0.2760885953903198,
+      -0.3088013529777527
+    ],
+    [
+      0.7659013271331787,
+      0.45173180103302,
+      0.08363968878984451
+    ],
+    [
+      0.046301186084747314,
+      0.32174208760261536,
+      -0.02479768544435501
+    ],
+    [
+      0.8852702379226685,
+      0.5638230443000793,
+      -0.30163514614105225
+    ],
+    [
+      0.14396211504936218,
+      -0.5229862928390503,
+      0.7057915329933167
+    ],
+    [
+      0.25057339668273926,
+      -0.48908960819244385,
+      1.2390971183776855
+    ],
+    [
+      0.9305946826934814,
+      -0.06769276410341263,
+      0.32944589853286743
+    ],
+    [
+      0.49603885412216187,
+      -0.30121496319770813,
+      0.11673831939697266
+    ],
+    [
+      0.18805721402168274,
+      -0.6352501511573792,
+      1.029891014099121
+    ],
+    [
+      0.4380909502506256,
+      0.2760885953903198,
+      -0.5105477571487427
+    ],
+    [
+      0.7968423962593079,
+      0.6381044983863831,
+      0.9040250182151794
+    ],
+    [
+      0.10855545848608017,
+      -0.07532769441604614,
+      0.5506806969642639
+    ],
+    [
+      0.8776094317436218,
+      0.5949564576148987,
+      0.5921839475631714
+    ],
+    [
+      0.8343055844306946,
+      -0.19943562150001526,
+      -0.45461326837539673
+    ],
+    [
+      0.7808497548103333,
+      0.8757853507995605,
+      -0.8390427827835083
+    ],
+    [
+      -0.13507428765296936,
+      -0.09689758718013763,
+      -0.30007806420326233
+    ],
+    [
+      0.3614951968193054,
+      -0.09229496866464615,
+      -0.3163415491580963
+    ],
+    [
+      0.4981468617916107,
+      -0.17805758118629456,
+      -0.4263595938682556
+    ],
+    [
+      0.6340190768241882,
+      0.2760885953903198,
+      0.683612585067749
+    ],
+    [
+      0.15833206474781036,
+      1.4134565591812134,
+      0.9688223004341125
+    ],
+    [
+      -0.1588582545518875,
+      -0.22736139595508575,
+      0.3688794672489166
+    ],
+    [
+      0.422553151845932,
+      0.07657118141651154,
+      0.1865272969007492
+    ],
+    [
+      -1.1082457304000854,
+      -1.1055575609207153,
+      -0.31395450234413147
+    ],
+    [
+      0.7734058499336243,
+      0.6510885953903198,
+      0.1986074447631836
+    ],
+    [
+      0.30664172768592834,
+      0.9047883749008179,
+      1.6732347011566162
+    ],
+    [
+      0.47173500061035156,
+      -0.404296338558197,
+      -0.05269849672913551
+    ],
+    [
+      0.2254696786403656,
+      1.0311392545700073,
+      0.7544334530830383
+    ],
+    [
+      0.28964507579803467,
+      0.2760885953903198,
+      0.004315612372010946
+    ],
+    [
+      -0.09603432565927505,
+      -0.1975087821483612,
+      0.2750057280063629
+    ],
+    [
+      0.6313948631286621,
+      -0.45082297921180725,
+      -1.0247089862823486
+    ],
+    [
+      0.6436291933059692,
+      0.7655327320098877,
+      1.5612726211547852
+    ],
+    [
+      1.6242343187332153,
+      0.5669596195220947,
+      1.3908542394638062
+    ],
+    [
+      1.6540242433547974,
+      0.017672497779130936,
+      -0.09003063291311264
+    ],
+    [
+      0.03177133947610855,
+      -0.0582880936563015,
+      0.30388104915618896
+    ],
+    [
+      0.4340118169784546,
+      0.22408443689346313,
+      0.42536166310310364
+    ],
+    [
+      -0.2615712881088257,
+      0.8843759894371033,
+      0.6420990824699402
+    ],
+    [
+      -0.4267977476119995,
+      0.2760885953903198,
+      1.2808101177215576
+    ],
+    [
+      0.25941160321235657,
+      -0.28659558296203613,
+      0.12422573566436768
+    ],
+    [
+      -0.017921442165970802,
+      0.5052063465118408,
+      0.8327353596687317
+    ],
+    [
+      1.43748140335083,
+      1.1108524799346924,
+      1.1908003091812134
+    ],
+    [
+      -0.21034519374370575,
+      -1.1540242433547974,
+      -0.3552939295768738
+    ],
+    [
+      1.6364960670471191,
+      -1.4828704595565796,
+      0.5128681659698486
+    ],
+    [
+      1.1857939958572388,
+      -0.2047322392463684,
+      -0.11171846836805344
+    ],
+    [
+      0.532031774520874,
+      0.47489693760871887,
+      -0.6211866736412048
+    ],
+    [
+      0.6138414144515991,
+      -0.12508337199687958,
+      0.8251534104347229
+    ],
+    [
+      1.0432484149932861,
+      -0.269777774810791,
+      -0.841098427772522
+    ],
+    [
+      0.6324758529663086,
+      -0.6827632188796997,
+      0.6175608038902283
+    ],
+    [
+      0.837646484375,
+      -0.4731529653072357,
+      0.4919143319129944
+    ],
+    [
+      0.7589465379714966,
+      -0.1913473904132843,
+      -0.04492336884140968
+    ],
+    [
+      1.675577163696289,
+      0.4276125431060791,
+      -0.08663908392190933
+    ],
+    [
+      1.3731650114059448,
+      0.31190672516822815,
+      0.8848100900650024
+    ],
+    [
+      0.5252143740653992,
+      0.33510568737983704,
+      0.24635496735572815
+    ],
+    [
+      -0.7712590098381042,
+      0.12278282642364502,
+      -0.23622973263263702
+    ],
+    [
+      0.6214934587478638,
+      -0.7585446834564209,
+      -0.1378912478685379
+    ],
+    [
+      -0.6390859484672546,
+      0.2760885953903198,
+      1.2160769701004028
+    ],
+    [
+      -0.3560261130332947,
+      0.827130913734436,
+      -0.2514520585536957
+    ],
+    [
+      0.3730946481227875,
+      -0.6099748015403748,
+      1.659493088722229
+    ],
+    [
+      1.271887183189392,
+      -0.7270786166191101,
+      -0.32162564992904663
+    ],
+    [
+      1.1162410974502563,
+      1.2644023895263672,
+      -0.30154871940612793
+    ],
+    [
+      1.0326673984527588,
+      -0.26783257722854614,
+      0.12563005089759827
+    ],
+    [
+      0.5051689147949219,
+      -0.4150272309780121,
+      -0.30528852343559265
+    ],
+    [
+      0.7575520873069763,
+      0.6434924006462097,
+      0.8242253065109253
+    ],
+    [
+      0.8129346966743469,
+      0.14466489851474762,
+      -0.13357996940612793
+    ],
+    [
+      1.302604079246521,
+      0.7171463966369629,
+      0.4617089033126831
+    ],
+    [
+      0.742294192314148,
+      -0.48782020807266235,
+      1.065754771232605
+    ],
+    [
+      1.012603759765625,
+      0.07203534245491028,
+      0.1455085128545761
+    ],
+    [
+      0.6692830324172974,
+      0.5570880770683289,
+      0.567864716053009
+    ],
+    [
+      1.6776037216186523,
+      -0.6200830340385437,
+      0.19839094579219818
+    ],
+    [
+      0.019362779334187508,
+      -0.5084202885627747,
+      -0.3302956819534302
+    ],
+    [
+      0.6852827668190002,
+      -0.2384236603975296,
+      0.11471696197986603
+    ],
+    [
+      0.6395102739334106,
+      0.6078513860702515,
+      -0.026797940954566002
+    ],
+    [
+      0.10872703790664673,
+      0.736443281173706,
+      0.4654095470905304
+    ],
+    [
+      1.3255295753479004,
+      1.2901142835617065,
+      -0.8800119757652283
+    ],
+    [
+      0.10669956356287003,
+      -0.4109380841255188,
+      0.5462028980255127
+    ],
+    [
+      0.2491900771856308,
+      -0.3824453353881836,
+      0.5839877128601074
+    ],
+    [
+      0.9604686498641968,
+      0.3857903480529785,
+      0.11673714220523834
+    ],
+    [
+      0.975523054599762,
+      -0.9491812586784363,
+      0.3749466836452484
+    ],
+    [
+      0.9288044571876526,
+      0.008424242027103901,
+      0.23254306614398956
+    ],
+    [
+      -0.959233283996582,
+      -0.3305145502090454,
+      0.36644890904426575
+    ],
+    [
+      -0.6329862475395203,
+      -1.0594964027404785,
+      0.9346150755882263
+    ],
+    [
+      0.5258339643478394,
+      1.431447982788086,
+      0.1422715038061142
+    ],
+    [
+      1.3395456075668335,
+      0.2760885953903198,
+      0.42089125514030457
+    ],
+    [
+      0.8835410475730896,
+      -0.1526731550693512,
+      -0.3783515989780426
+    ],
+    [
+      0.06484033167362213,
+      -0.5363202691078186,
+      -0.8604055047035217
+    ],
+    [
+      0.04306585341691971,
+      0.9516770839691162,
+      0.5557815432548523
+    ],
+    [
+      1.327269196510315,
+      -0.26236361265182495,
+      0.31885039806365967
+    ],
+    [
+      0.5059521198272705,
+      -1.0002529621124268,
+      -0.5267103314399719
+    ],
+    [
+      0.029395291581749916,
+      0.9818071126937866,
+      0.14862196147441864
+    ],
+    [
+      0.11171174049377441,
+      -0.13735109567642212,
+      -0.0008401983650401235
+    ],
+    [
+      1.2116097211837769,
+      0.8473241329193115,
+      0.88216632604599
+    ],
+    [
+      -0.6192097663879395,
+      0.2760885953903198,
+      0.1587342619895935
+    ],
+    [
+      0.6281998753547668,
+      0.5921935439109802,
+      0.17157454788684845
+    ],
+    [
+      -0.3155535161495209,
+      -0.5330051779747009,
+      -0.059077147394418716
+    ],
+    [
+      0.20091448724269867,
+      -0.5068090558052063,
+      -0.18025846779346466
+    ],
+    [
+      2.7466068267822266,
+      -0.8748731017112732,
+      -0.04711220785975456
+    ],
+    [
+      1.447810411453247,
+      -0.27255988121032715,
+      -0.2962985336780548
+    ],
+    [
+      1.60551118850708,
+      -0.11915700882673264,
+      -0.5255311131477356
+    ],
+    [
+      -0.887458324432373,
+      0.8258630633354187,
+      0.30738890171051025
+    ],
+    [
+      2.456341028213501,
+      1.016897439956665,
+      0.7945164442062378
+    ],
+    [
+      -0.2242172211408615,
+      0.8882625102996826,
+      1.1776946783065796
+    ],
+    [
+      -0.39268550276756287,
+      -0.4738922119140625,
+      1.6032382249832153
+    ],
+    [
+      1.8086175918579102,
+      -1.2800241708755493,
+      -0.9058151245117188
+    ],
+    [
+      0.2887733578681946,
+      -0.7111950516700745,
+      0.6073051691055298
+    ],
+    [
+      -1.3280118703842163,
+      -1.6134986877441406,
+      1.5988401174545288
+    ],
+    [
+      2.2223989963531494,
+      -1.8312017917633057,
+      0.9205300211906433
+    ],
+    [
+      0.19130605459213257,
+      -0.6441234946250916,
+      -0.5636095404624939
+    ],
+    [
+      -0.2631304860115051,
+      0.0504564493894577,
+      -0.6263415813446045
+    ],
+    [
+      2.096590518951416,
+      -0.8093783259391785,
+      0.659914493560791
+    ],
+    [
+      -0.6602017879486084,
+      0.2760885953903198,
+      1.6043192148208618
+    ],
+    [
+      -1.4940307140350342,
+      -1.2088669538497925,
+      1.6487345695495605
+    ],
+    [
+      -0.09679925441741943,
+      0.4763238728046417,
+      -0.4205774664878845
+    ],
+    [
+      0.7659120559692383,
+      0.9064326882362366,
+      0.11784222722053528
+    ],
+    [
+      1.7383723258972168,
+      -1.3881635665893555,
+      0.7730268239974976
+    ],
+    [
+      1.1018028259277344,
+      -0.5354564785957336,
+      0.7790213823318481
+    ],
+    [
+      1.936551809310913,
+      -0.9125238656997681,
+      -0.49167728424072266
+    ],
+    [
+      -0.5996152758598328,
+      -0.8539415597915649,
+      0.33622652292251587
+    ],
+    [
+      0.6356623768806458,
+      -1.479321002960205,
+      1.1427593231201172
+    ],
+    [
+      -0.7094821929931641,
+      1.060670256614685,
+      1.5971366167068481
+    ],
+    [
+      -0.4047229588031769,
+      -0.5818800926208496,
+      1.0588374137878418
+    ],
+    [
+      0.21128810942173004,
+      -0.625886857509613,
+      0.23703314363956451
+    ],
+    [
+      0.15707284212112427,
+      0.30370190739631653,
+      0.8887234926223755
+    ],
+    [
+      -0.05224579945206642,
+      -1.3394489288330078,
+      -0.40957173705101013
+    ],
+    [
+      0.5565025210380554,
+      0.34809643030166626,
+      -0.311869740486145
+    ],
+    [
+      -2.1316611766815186,
+      -0.11549966037273407,
+      1.518341302871704
+    ],
+    [
+      -2.1973228454589844,
+      -0.11372040212154388,
+      0.30949142575263977
+    ],
+    [
+      -0.6867638826370239,
+      0.735841691493988,
+      0.4329794645309448
+    ],
+    [
+      -0.26326698064804077,
+      0.2760885953903198,
+      -0.5323948264122009
+    ],
+    [
+      1.0033975839614868,
+      0.2760885953903198,
+      -0.4779791235923767
+    ],
+    [
+      2.5320513248443604,
+      -1.1507349014282227,
+      -2.4573686122894287
+    ],
+    [
+      -1.9798444509506226,
+      3.7906363010406494,
+      1.0800453424453735
+    ]
+  ],
+  "model_names": [
+    "instruct",
+    "math",
+    "code"
+  ],
+  "num_models": 3,
+  "num_params": 291,
+  "param_names": [
+    "model.embed_tokens.weight",
+    "model.layers.0.self_attn.q_proj.weight",
+    "model.layers.0.self_attn.k_proj.weight",
+    "model.layers.0.self_attn.v_proj.weight",
+    "model.layers.0.self_attn.o_proj.weight",
+    "model.layers.0.mlp.gate_proj.weight",
+    "model.layers.0.mlp.up_proj.weight",
+    "model.layers.0.mlp.down_proj.weight",
+    "model.layers.0.input_layernorm.weight",
+    "model.layers.0.post_attention_layernorm.weight",
+    "model.layers.1.self_attn.q_proj.weight",
+    "model.layers.1.self_attn.k_proj.weight",
+    "model.layers.1.self_attn.v_proj.weight",
+    "model.layers.1.self_attn.o_proj.weight",
+    "model.layers.1.mlp.gate_proj.weight",
+    "model.layers.1.mlp.up_proj.weight",
+    "model.layers.1.mlp.down_proj.weight",
+    "model.layers.1.input_layernorm.weight",
+    "model.layers.1.post_attention_layernorm.weight",
+    "model.layers.2.self_attn.q_proj.weight",
+    "model.layers.2.self_attn.k_proj.weight",
+    "model.layers.2.self_attn.v_proj.weight",
+    "model.layers.2.self_attn.o_proj.weight",
+    "model.layers.2.mlp.gate_proj.weight",
+    "model.layers.2.mlp.up_proj.weight",
+    "model.layers.2.mlp.down_proj.weight",
+    "model.layers.2.input_layernorm.weight",
+    "model.layers.2.post_attention_layernorm.weight",
+    "model.layers.3.self_attn.q_proj.weight",
+    "model.layers.3.self_attn.k_proj.weight",
+    "model.layers.3.self_attn.v_proj.weight",
+    "model.layers.3.self_attn.o_proj.weight",
+    "model.layers.3.mlp.gate_proj.weight",
+    "model.layers.3.mlp.up_proj.weight",
+    "model.layers.3.mlp.down_proj.weight",
+    "model.layers.3.input_layernorm.weight",
+    "model.layers.3.post_attention_layernorm.weight",
+    "model.layers.4.self_attn.q_proj.weight",
+    "model.layers.4.self_attn.k_proj.weight",
+    "model.layers.4.self_attn.v_proj.weight",
+    "model.layers.4.self_attn.o_proj.weight",
+    "model.layers.4.mlp.gate_proj.weight",
+    "model.layers.4.mlp.up_proj.weight",
+    "model.layers.4.mlp.down_proj.weight",
+    "model.layers.4.input_layernorm.weight",
+    "model.layers.4.post_attention_layernorm.weight",
+    "model.layers.5.self_attn.q_proj.weight",
+    "model.layers.5.self_attn.k_proj.weight",
+    "model.layers.5.self_attn.v_proj.weight",
+    "model.layers.5.self_attn.o_proj.weight",
+    "model.layers.5.mlp.gate_proj.weight",
+    "model.layers.5.mlp.up_proj.weight",
+    "model.layers.5.mlp.down_proj.weight",
+    "model.layers.5.input_layernorm.weight",
+    "model.layers.5.post_attention_layernorm.weight",
+    "model.layers.6.self_attn.q_proj.weight",
+    "model.layers.6.self_attn.k_proj.weight",
+    "model.layers.6.self_attn.v_proj.weight",
+    "model.layers.6.self_attn.o_proj.weight",
+    "model.layers.6.mlp.gate_proj.weight",
+    "model.layers.6.mlp.up_proj.weight",
+    "model.layers.6.mlp.down_proj.weight",
+    "model.layers.6.input_layernorm.weight",
+    "model.layers.6.post_attention_layernorm.weight",
+    "model.layers.7.self_attn.q_proj.weight",
+    "model.layers.7.self_attn.k_proj.weight",
+    "model.layers.7.self_attn.v_proj.weight",
+    "model.layers.7.self_attn.o_proj.weight",
+    "model.layers.7.mlp.gate_proj.weight",
+    "model.layers.7.mlp.up_proj.weight",
+    "model.layers.7.mlp.down_proj.weight",
+    "model.layers.7.input_layernorm.weight",
+    "model.layers.7.post_attention_layernorm.weight",
+    "model.layers.8.self_attn.q_proj.weight",
+    "model.layers.8.self_attn.k_proj.weight",
+    "model.layers.8.self_attn.v_proj.weight",
+    "model.layers.8.self_attn.o_proj.weight",
+    "model.layers.8.mlp.gate_proj.weight",
+    "model.layers.8.mlp.up_proj.weight",
+    "model.layers.8.mlp.down_proj.weight",
+    "model.layers.8.input_layernorm.weight",
+    "model.layers.8.post_attention_layernorm.weight",
+    "model.layers.9.self_attn.q_proj.weight",
+    "model.layers.9.self_attn.k_proj.weight",
+    "model.layers.9.self_attn.v_proj.weight",
+    "model.layers.9.self_attn.o_proj.weight",
+    "model.layers.9.mlp.gate_proj.weight",
+    "model.layers.9.mlp.up_proj.weight",
+    "model.layers.9.mlp.down_proj.weight",
+    "model.layers.9.input_layernorm.weight",
+    "model.layers.9.post_attention_layernorm.weight",
+    "model.layers.10.self_attn.q_proj.weight",
+    "model.layers.10.self_attn.k_proj.weight",
+    "model.layers.10.self_attn.v_proj.weight",
+    "model.layers.10.self_attn.o_proj.weight",
+    "model.layers.10.mlp.gate_proj.weight",
+    "model.layers.10.mlp.up_proj.weight",
+    "model.layers.10.mlp.down_proj.weight",
+    "model.layers.10.input_layernorm.weight",
+    "model.layers.10.post_attention_layernorm.weight",
+    "model.layers.11.self_attn.q_proj.weight",
+    "model.layers.11.self_attn.k_proj.weight",
+    "model.layers.11.self_attn.v_proj.weight",
+    "model.layers.11.self_attn.o_proj.weight",
+    "model.layers.11.mlp.gate_proj.weight",
+    "model.layers.11.mlp.up_proj.weight",
+    "model.layers.11.mlp.down_proj.weight",
+    "model.layers.11.input_layernorm.weight",
+    "model.layers.11.post_attention_layernorm.weight",
+    "model.layers.12.self_attn.q_proj.weight",
+    "model.layers.12.self_attn.k_proj.weight",
+    "model.layers.12.self_attn.v_proj.weight",
+    "model.layers.12.self_attn.o_proj.weight",
+    "model.layers.12.mlp.gate_proj.weight",
+    "model.layers.12.mlp.up_proj.weight",
+    "model.layers.12.mlp.down_proj.weight",
+    "model.layers.12.input_layernorm.weight",
+    "model.layers.12.post_attention_layernorm.weight",
+    "model.layers.13.self_attn.q_proj.weight",
+    "model.layers.13.self_attn.k_proj.weight",
+    "model.layers.13.self_attn.v_proj.weight",
+    "model.layers.13.self_attn.o_proj.weight",
+    "model.layers.13.mlp.gate_proj.weight",
+    "model.layers.13.mlp.up_proj.weight",
+    "model.layers.13.mlp.down_proj.weight",
+    "model.layers.13.input_layernorm.weight",
+    "model.layers.13.post_attention_layernorm.weight",
+    "model.layers.14.self_attn.q_proj.weight",
+    "model.layers.14.self_attn.k_proj.weight",
+    "model.layers.14.self_attn.v_proj.weight",
+    "model.layers.14.self_attn.o_proj.weight",
+    "model.layers.14.mlp.gate_proj.weight",
+    "model.layers.14.mlp.up_proj.weight",
+    "model.layers.14.mlp.down_proj.weight",
+    "model.layers.14.input_layernorm.weight",
+    "model.layers.14.post_attention_layernorm.weight",
+    "model.layers.15.self_attn.q_proj.weight",
+    "model.layers.15.self_attn.k_proj.weight",
+    "model.layers.15.self_attn.v_proj.weight",
+    "model.layers.15.self_attn.o_proj.weight",
+    "model.layers.15.mlp.gate_proj.weight",
+    "model.layers.15.mlp.up_proj.weight",
+    "model.layers.15.mlp.down_proj.weight",
+    "model.layers.15.input_layernorm.weight",
+    "model.layers.15.post_attention_layernorm.weight",
+    "model.layers.16.self_attn.q_proj.weight",
+    "model.layers.16.self_attn.k_proj.weight",
+    "model.layers.16.self_attn.v_proj.weight",
+    "model.layers.16.self_attn.o_proj.weight",
+    "model.layers.16.mlp.gate_proj.weight",
+    "model.layers.16.mlp.up_proj.weight",
+    "model.layers.16.mlp.down_proj.weight",
+    "model.layers.16.input_layernorm.weight",
+    "model.layers.16.post_attention_layernorm.weight",
+    "model.layers.17.self_attn.q_proj.weight",
+    "model.layers.17.self_attn.k_proj.weight",
+    "model.layers.17.self_attn.v_proj.weight",
+    "model.layers.17.self_attn.o_proj.weight",
+    "model.layers.17.mlp.gate_proj.weight",
+    "model.layers.17.mlp.up_proj.weight",
+    "model.layers.17.mlp.down_proj.weight",
+    "model.layers.17.input_layernorm.weight",
+    "model.layers.17.post_attention_layernorm.weight",
+    "model.layers.18.self_attn.q_proj.weight",
+    "model.layers.18.self_attn.k_proj.weight",
+    "model.layers.18.self_attn.v_proj.weight",
+    "model.layers.18.self_attn.o_proj.weight",
+    "model.layers.18.mlp.gate_proj.weight",
+    "model.layers.18.mlp.up_proj.weight",
+    "model.layers.18.mlp.down_proj.weight",
+    "model.layers.18.input_layernorm.weight",
+    "model.layers.18.post_attention_layernorm.weight",
+    "model.layers.19.self_attn.q_proj.weight",
+    "model.layers.19.self_attn.k_proj.weight",
+    "model.layers.19.self_attn.v_proj.weight",
+    "model.layers.19.self_attn.o_proj.weight",
+    "model.layers.19.mlp.gate_proj.weight",
+    "model.layers.19.mlp.up_proj.weight",
+    "model.layers.19.mlp.down_proj.weight",
+    "model.layers.19.input_layernorm.weight",
+    "model.layers.19.post_attention_layernorm.weight",
+    "model.layers.20.self_attn.q_proj.weight",
+    "model.layers.20.self_attn.k_proj.weight",
+    "model.layers.20.self_attn.v_proj.weight",
+    "model.layers.20.self_attn.o_proj.weight",
+    "model.layers.20.mlp.gate_proj.weight",
+    "model.layers.20.mlp.up_proj.weight",
+    "model.layers.20.mlp.down_proj.weight",
+    "model.layers.20.input_layernorm.weight",
+    "model.layers.20.post_attention_layernorm.weight",
+    "model.layers.21.self_attn.q_proj.weight",
+    "model.layers.21.self_attn.k_proj.weight",
+    "model.layers.21.self_attn.v_proj.weight",
+    "model.layers.21.self_attn.o_proj.weight",
+    "model.layers.21.mlp.gate_proj.weight",
+    "model.layers.21.mlp.up_proj.weight",
+    "model.layers.21.mlp.down_proj.weight",
+    "model.layers.21.input_layernorm.weight",
+    "model.layers.21.post_attention_layernorm.weight",
+    "model.layers.22.self_attn.q_proj.weight",
+    "model.layers.22.self_attn.k_proj.weight",
+    "model.layers.22.self_attn.v_proj.weight",
+    "model.layers.22.self_attn.o_proj.weight",
+    "model.layers.22.mlp.gate_proj.weight",
+    "model.layers.22.mlp.up_proj.weight",
+    "model.layers.22.mlp.down_proj.weight",
+    "model.layers.22.input_layernorm.weight",
+    "model.layers.22.post_attention_layernorm.weight",
+    "model.layers.23.self_attn.q_proj.weight",
+    "model.layers.23.self_attn.k_proj.weight",
+    "model.layers.23.self_attn.v_proj.weight",
+    "model.layers.23.self_attn.o_proj.weight",
+    "model.layers.23.mlp.gate_proj.weight",
+    "model.layers.23.mlp.up_proj.weight",
+    "model.layers.23.mlp.down_proj.weight",
+    "model.layers.23.input_layernorm.weight",
+    "model.layers.23.post_attention_layernorm.weight",
+    "model.layers.24.self_attn.q_proj.weight",
+    "model.layers.24.self_attn.k_proj.weight",
+    "model.layers.24.self_attn.v_proj.weight",
+    "model.layers.24.self_attn.o_proj.weight",
+    "model.layers.24.mlp.gate_proj.weight",
+    "model.layers.24.mlp.up_proj.weight",
+    "model.layers.24.mlp.down_proj.weight",
+    "model.layers.24.input_layernorm.weight",
+    "model.layers.24.post_attention_layernorm.weight",
+    "model.layers.25.self_attn.q_proj.weight",
+    "model.layers.25.self_attn.k_proj.weight",
+    "model.layers.25.self_attn.v_proj.weight",
+    "model.layers.25.self_attn.o_proj.weight",
+    "model.layers.25.mlp.gate_proj.weight",
+    "model.layers.25.mlp.up_proj.weight",
+    "model.layers.25.mlp.down_proj.weight",
+    "model.layers.25.input_layernorm.weight",
+    "model.layers.25.post_attention_layernorm.weight",
+    "model.layers.26.self_attn.q_proj.weight",
+    "model.layers.26.self_attn.k_proj.weight",
+    "model.layers.26.self_attn.v_proj.weight",
+    "model.layers.26.self_attn.o_proj.weight",
+    "model.layers.26.mlp.gate_proj.weight",
+    "model.layers.26.mlp.up_proj.weight",
+    "model.layers.26.mlp.down_proj.weight",
+    "model.layers.26.input_layernorm.weight",
+    "model.layers.26.post_attention_layernorm.weight",
+    "model.layers.27.self_attn.q_proj.weight",
+    "model.layers.27.self_attn.k_proj.weight",
+    "model.layers.27.self_attn.v_proj.weight",
+    "model.layers.27.self_attn.o_proj.weight",
+    "model.layers.27.mlp.gate_proj.weight",
+    "model.layers.27.mlp.up_proj.weight",
+    "model.layers.27.mlp.down_proj.weight",
+    "model.layers.27.input_layernorm.weight",
+    "model.layers.27.post_attention_layernorm.weight",
+    "model.layers.28.self_attn.q_proj.weight",
+    "model.layers.28.self_attn.k_proj.weight",
+    "model.layers.28.self_attn.v_proj.weight",
+    "model.layers.28.self_attn.o_proj.weight",
+    "model.layers.28.mlp.gate_proj.weight",
+    "model.layers.28.mlp.up_proj.weight",
+    "model.layers.28.mlp.down_proj.weight",
+    "model.layers.28.input_layernorm.weight",
+    "model.layers.28.post_attention_layernorm.weight",
+    "model.layers.29.self_attn.q_proj.weight",
+    "model.layers.29.self_attn.k_proj.weight",
+    "model.layers.29.self_attn.v_proj.weight",
+    "model.layers.29.self_attn.o_proj.weight",
+    "model.layers.29.mlp.gate_proj.weight",
+    "model.layers.29.mlp.up_proj.weight",
+    "model.layers.29.mlp.down_proj.weight",
+    "model.layers.29.input_layernorm.weight",
+    "model.layers.29.post_attention_layernorm.weight",
+    "model.layers.30.self_attn.q_proj.weight",
+    "model.layers.30.self_attn.k_proj.weight",
+    "model.layers.30.self_attn.v_proj.weight",
+    "model.layers.30.self_attn.o_proj.weight",
+    "model.layers.30.mlp.gate_proj.weight",
+    "model.layers.30.mlp.up_proj.weight",
+    "model.layers.30.mlp.down_proj.weight",
+    "model.layers.30.input_layernorm.weight",
+    "model.layers.30.post_attention_layernorm.weight",
+    "model.layers.31.self_attn.q_proj.weight",
+    "model.layers.31.self_attn.k_proj.weight",
+    "model.layers.31.self_attn.v_proj.weight",
+    "model.layers.31.self_attn.o_proj.weight",
+    "model.layers.31.mlp.gate_proj.weight",
+    "model.layers.31.mlp.up_proj.weight",
+    "model.layers.31.mlp.down_proj.weight",
+    "model.layers.31.input_layernorm.weight",
+    "model.layers.31.post_attention_layernorm.weight",
+    "model.norm.weight",
+    "lm_head.weight"
+  ]
+}

logs/save_merged_model_20250703_172454.log

@@ -0,0 +1,74 @@
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO - Starting merged model save process
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO - Arguments: {'lambdas_path': '/work/gj26/b20042/LLM-AdaMerge/outputs/mistral-7b/parameter-wise/test-time-adaptation-large-lr/llm_adamerge_parameterwise_lambdas.json', 'model_config': '/work/gj26/b20042/LLM-AdaMerge/src/configs/model_config.yaml', 'output_dir': '/work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-time-adaptation-large-lr', 'model_name': 'merged-model', 'push_to_hub': False, 'hub_repo_id': 'lejelly/test-time-adaptation-ep3-lr0.01-llm-adamerge-shannonentropy-mistral-7b-instrcut-math-code', 'private': False, 'device': 'cuda', 'debug': False}
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO - Loading lambdas from /work/gj26/b20042/LLM-AdaMerge/outputs/mistral-7b/parameter-wise/test-time-adaptation-large-lr/llm_adamerge_parameterwise_lambdas.json
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO - Auto-detected parameter-wise merge from JSON structure
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO - Merge type: parameter_wise
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO - [Initial] Memory Usage:
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO -   Process: 0.37 GB (0.2%)
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO -   System: 8.98 GB / 212.49 GB (8.9%)
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO -   Available: 193.65 GB
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-07-03 17:24:54 - experiment_save_merged_model - INFO - Loading models
+2025-07-03 17:25:13 - experiment_save_merged_model - INFO - [After loading models] Memory Usage:
+2025-07-03 17:25:13 - experiment_save_merged_model - INFO -   Process: 40.95 GB (19.3%)
+2025-07-03 17:25:13 - experiment_save_merged_model - INFO -   System: 48.99 GB / 212.49 GB (31.0%)
+2025-07-03 17:25:13 - experiment_save_merged_model - INFO -   Available: 146.62 GB
+2025-07-03 17:25:13 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-07-03 17:25:13 - experiment_save_merged_model - INFO - Initializing parameter_wise AdaMerge
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - Loading learned lambdas
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - Deleting original models to free memory (task vectors already computed)
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - [Before deleting models] Memory Usage:
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   Process: 95.03 GB (44.7%)
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   System: 90.00 GB / 212.49 GB (50.3%)
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   Available: 105.56 GB
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - Clearing model_loader references
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - Deleting model variables
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - Running garbage collection
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - [After deleting models and GC] Memory Usage:
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   Process: 55.70 GB (26.2%)
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   System: 64.02 GB / 212.49 GB (38.1%)
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   Available: 131.53 GB
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - [After loading lambdas] Memory Usage:
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   Process: 55.70 GB (26.2%)
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   System: 64.02 GB / 212.49 GB (38.1%)
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   Available: 131.53 GB
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - Creating merged model with learned lambdas
+2025-07-03 17:26:28 - experiment_save_merged_model - INFO - Using merge_models_for_save()
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO - [After merging models] Memory Usage:
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO -   Process: 57.68 GB (27.1%)
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO -   System: 93.06 GB / 212.49 GB (48.6%)
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO -   Available: 109.21 GB
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 13.49 GB, Reserved: 27.23 GB, Total: 94.50 GB
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO - Freeing memory from AdaMerge object (task vectors and base params no longer needed)
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO - Deleting task vectors
+2025-07-03 17:28:29 - experiment_save_merged_model - INFO - Deleting base params
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO - Deleting functional model
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO - [After freeing AdaMerge memory] Memory Usage:
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO -   Process: 5.69 GB (2.7%)
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO -   System: 27.34 GB / 212.49 GB (17.7%)
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO -   Available: 174.93 GB
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 13.49 GB, Reserved: 13.62 GB, Total: 94.50 GB
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO - Saving merged model to /work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-time-adaptation-large-lr
+2025-07-03 17:28:30 - experiment_save_merged_model - INFO - Moving merged model to CPU for saving
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO - Successfully saved 3 safetensors files:
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO -   - model-00001-of-00003.safetensors (4714.17 MB)
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO -   - model-00003-of-00003.safetensors (4330.17 MB)
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO -   - model-00002-of-00003.safetensors (4768.20 MB)
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO - [After saving model] Memory Usage:
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO -   Process: 15.34 GB (7.2%)
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO -   System: 23.64 GB / 212.49 GB (19.1%)
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO -   Available: 171.93 GB
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO - Saving tokenizer
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO - Copied lambdas file to /work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-time-adaptation-large-lr/learned_lambdas.json
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO - Creating model card
+2025-07-03 17:29:09 - experiment_save_merged_model - INFO - Cleaning up models
+2025-07-03 17:29:10 - experiment_save_merged_model - INFO - [After cleanup] Memory Usage:
+2025-07-03 17:29:10 - experiment_save_merged_model - INFO -   Process: 15.34 GB (7.2%)
+2025-07-03 17:29:10 - experiment_save_merged_model - INFO -   System: 23.64 GB / 212.49 GB (19.1%)
+2025-07-03 17:29:10 - experiment_save_merged_model - INFO -   Available: 171.92 GB
+2025-07-03 17:29:10 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-07-03 17:29:10 - experiment_save_merged_model - INFO - Model saved successfully to /work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-time-adaptation-large-lr

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e5409526e2583e5865b5224c0c3a9910979df310e0173a73fe7e6a1db724996
+size 4943162240

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4f2864fec9281d27c761fa2458bf19e7d249979999a76814d69f0bdfa790e76
+size 4999819232

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82b5a5daadc9879e06ac5114e7ac805597223ea6317db0006bdd2fc286639ad9
+size 4540516256

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizerFast",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}