diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,3865 @@ +{ + "metadata": { + "ParamSize": 339, + "ParamBytes": 1075777536.0, + "BitsPerParam": 5.001944354676003 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 155582464, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 151936, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 155582464, + "byteOffset": 0 + } + ], + "md5sum": "c738b02f2921806bec6241c056c7d89f" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 26529792, + "records": [ + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 151936, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 19447808, + "byteOffset": 0 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 19447808 + }, + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 19451904 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 25743360 + } + ], + "md5sum": "f05196d49032d9c2e94030aff0347ddb" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.0.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.0.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "58fa452a734fa6344be8d04ab2ac3530" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.1.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.1.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "35901d5fdc663e6eb983927cf00d9d40" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.10.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.10.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "ab8fe57de6e885c25e106514607d88a2" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.11.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.11.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "db03f10590f172e0f8f2f8b38cd523b4" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.12.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.12.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "cadb334a9f3c929e0807d55c911887b7" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.13.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.13.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "3044f156e8c7afcc8193636604115952" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.14.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.14.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "4331c0a63de68ddadaac1a5e8e091b41" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.15.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.15.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "66ef186e333bf1ef9c83e310f98617d8" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.16.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.16.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "551b65feddb0cb8922afc06f69dc61e9" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.17.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.17.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "a899def09baf93aaea417059d4922479" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.18.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.18.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "35c56ed6b4f6686b57910dc061c6168e" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.19.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.19.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "7e0f219c9790cbfb9ae2cc8648984c17" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.2.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.2.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "beb40a98c4fa149be2424daccd4a161f" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.20.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.20.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "050c3b29f1b8d620d6c40e9d486125fa" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.21.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.21.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "020174288427cfb03ece4e142c9fba2f" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.22.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.22.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "93b1c196dbcd0c4adf3f70bbf6fbdf6e" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.23.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.23.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.24.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.24.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "ca0527055bd57f4ab18d9f88a75f632c" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.24.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.24.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.24.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.24.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.24.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.25.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.25.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "9c006b8b2b1213001610a7adf3de46b0" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.25.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.25.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.25.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.26.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.26.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "66b3a362dc0e0d44be4f217bdc283082" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.26.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.26.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.26.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.27.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.27.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "4d08162385b66c99dc6ce9a1d015d49b" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.27.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.27.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.27.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "612f83f464f5a59100816af19df3956d" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.3.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.3.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "cb902c52d867dcb113537adb1379b994" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.4.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.4.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "627db0a648b8c6045de0d96427b33dfe" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.5.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.5.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "5babf98ba5f5afb2c893b675a6a98130" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.6.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.6.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "29304a813def1cb57aaef95cbe4c8135" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.7.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.7.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "ef5fc7e69a43af17081d3de057e1ae4b" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 28320256, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.8.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.8.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + }, + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 2048, + 768 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 21242368 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 2048, + 192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 27533824 + } + ], + "md5sum": "b160f2742774a842c966440c101be8cb" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 21242368, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 12288, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 12288, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 14155776 + }, + { + "name": "model.layers.9.self_attn.k_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 14159872 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_weight", + "shape": [ + 4096, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4194304, + "byteOffset": 14160128 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_scale", + "shape": [ + 4096, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 524288, + "byteOffset": 18354432 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 18878720 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 20975872 + }, + { + "name": "model.layers.9.self_attn.q_norm.weight", + "shape": [ + 128 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 21238016 + }, + { + "name": "model.norm.weight", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 21238272 + } + ], + "md5sum": "b702c0f588cad2e6d858461c3329dc64" + } + ] +} \ No newline at end of file