{ "metadata": { "ParamSize": 325, "ParamBytes": 1925183488.0, "BitsPerParam": 4.0306587953497806 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 49250304, "records": [ { "name": "lm_head.q_weight", "shape": [ 32064, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 49250304, "byteOffset": 0 } ], "md5sum": "5d3ade109e743cd5c15798661e7d8286" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.21.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "78a4a5b20f833cb7651edbe81a1a0d5e" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 32970752, "records": [ { "name": "lm_head.q_scale", "shape": [ 32064, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6156288, "byteOffset": 0 }, { "name": "transformer.h.21.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 6156288 }, { "name": "transformer.h.21.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 6162432 }, { "name": "transformer.h.21.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18745344 }, { "name": "transformer.h.21.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 18751488 }, { "name": "transformer.h.21.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18784256 }, { "name": "transformer.h.21.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 18790400 }, { "name": "transformer.h.21.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 32946176 }, { "name": "transformer.h.22.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 32964608 } ], "md5sum": "ec22f38e4e2eb3adcab232d2eefb787a" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.22.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "1d6767246ead182f0a023599dc4f3c81" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.22.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.22.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.22.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.22.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.22.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.22.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.22.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.22.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.23.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "a51cfe876723944c76711c90512ea8bd" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.23.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "1cb656f01b57771845f9d0933cfc6759" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.23.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.23.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.23.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.23.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.23.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.23.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.23.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.23.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.24.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "a3a1d182458f05aa629cdb7b6f345350" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.24.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "992765155769413002936cf7a9ea4442" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.24.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.24.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.24.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.24.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.24.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.24.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.24.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.24.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.25.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "b4f4b51b1bf9b62d435adf8315381b76" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.25.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "9f9adfa4f11cd1d1d0752107f1f0599c" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.25.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.25.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.25.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.25.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.25.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.25.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.25.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.25.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.26.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "b5f5e995de08a27639f925ec9b8190e4" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.26.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "12fc1ea95688f5cae1cf9aac3deb7be4" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.26.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.26.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.26.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.26.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.26.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.26.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.26.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.26.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.27.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "707c78d752687048395d4ab2f28ef091" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.27.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "9781ce88fb38c7010bfcf4fbfdf790c3" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.27.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.27.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.27.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.27.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.27.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.27.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.27.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.27.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.28.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "be7dabb40a3101ca81294834939c2a4f" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.28.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "2bb1fb07b2b23de1b3a2d154082f5563" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.28.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.28.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.28.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.28.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.28.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.28.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.28.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.28.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.29.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "570d9a59ec7712508eec93aa0bacdf8e" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.29.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "d4ed37b61694548dbfab578a073d7350" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.29.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.29.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.29.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.29.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.29.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.29.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.29.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.29.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.30.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "a0b172795ba6f3df91eff773779c5e09" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.30.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "5351f3145599b8e12503deee977d6939" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.30.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.30.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.30.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.30.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.30.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.30.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.30.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.30.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.h.31.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "c31d1ae59506dc162a160ef4db715395" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.31.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "28488ad3fdde10a98e057d9df8ab6916" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 49250304, "records": [ { "name": "transformer.embd.q_weight", "shape": [ 32064, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 49250304, "byteOffset": 0 } ], "md5sum": "3f9a3cb77fdaedfd62e9720fc14e716e" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.31.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.31.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12582912 }, { "name": "transformer.h.31.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 12589056 }, { "name": "transformer.h.31.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 12621824 }, { "name": "transformer.h.31.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 12627968 }, { "name": "transformer.h.31.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17346560 }, { "name": "transformer.h.31.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 17352704 }, { "name": "transformer.h.31.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 31508480 }, { "name": "transformer.norm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "a82cc1c4e8a3a7ebb02d781ae768de39" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.0.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "10d41991771e6254005c82a8e3d55795" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 23515136, "records": [ { "name": "transformer.embd.q_scale", "shape": [ 32064, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6156288, "byteOffset": 0 }, { "name": "transformer.h.0.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 6156288 }, { "name": "transformer.h.0.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 6162432 }, { "name": "transformer.h.0.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18745344 }, { "name": "transformer.h.0.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 18751488 }, { "name": "transformer.h.0.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18784256 }, { "name": "transformer.h.0.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 18790400 }, { "name": "transformer.h.0.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23508992 } ], "md5sum": "bf54fa3e7550c7f42036f02f0626d222" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.1.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "8a25aac59c6eeb5df903f883f7a5d540" }, { "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.0.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.0.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.1.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.1.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.1.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.1.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.1.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.1.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.1.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "9b4a683770c286c3e7d5e9120ef1d5dd" }, { "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.10.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "c34505f475449bb2c465b0a633a25793" }, { "dataPath": "params_shard_29.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.1.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.1.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.10.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.10.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.10.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.10.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.10.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.10.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.10.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "76885693cac42f873bb691ea6fb57b0c" }, { "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.11.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "7023c08ce4586f7fc873bbc6d1149d8e" }, { "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.10.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.10.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.11.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.11.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.11.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.11.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.11.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.11.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.11.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "ba20f39eec0167be058927c1e5aae82a" }, { "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.12.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "f9f2bf14df09314718d370f0e86f97b0" }, { "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.11.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.11.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.12.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.12.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.12.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.12.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.12.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.12.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.12.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "6dad8994e7265368eb895c8dba21715b" }, { "dataPath": "params_shard_34.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.13.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "30dda05e0880bdb98b4067d5f90baa8e" }, { "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.12.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.12.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.13.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.13.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.13.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.13.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.13.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.13.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.13.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "52bf5c91439431490af7a116788bea08" }, { "dataPath": "params_shard_36.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.14.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "f8e804a983415caca4e31df3047a6e1a" }, { "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.13.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.13.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.14.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.14.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.14.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.14.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.14.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.14.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.14.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "8d1484cbe9cad4e0c1e211b4af7b07ad" }, { "dataPath": "params_shard_38.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.15.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "09aa9091a48044a5f6e769ff74b9c8fa" }, { "dataPath": "params_shard_39.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.14.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.14.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.15.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.15.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.15.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.15.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.15.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.15.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.15.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "8dd2efe06dcb3ee1958064aa0f8d7dcc" }, { "dataPath": "params_shard_40.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.16.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "f9e0d79d96e92283a0cc0f4c8db060e1" }, { "dataPath": "params_shard_41.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.15.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.15.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.16.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.16.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.16.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.16.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.16.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.16.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.16.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "91c16d869e8b43e56fe011ddefdaca1d" }, { "dataPath": "params_shard_42.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.17.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "e9a5dd6b887e8aae0c5e162438da9365" }, { "dataPath": "params_shard_43.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.16.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.16.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.17.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.17.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.17.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.17.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.17.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.17.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.17.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "8784e6830067f7ca05e331976b50a12d" }, { "dataPath": "params_shard_44.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.18.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "102f3b65b0faa129450be3a420f43d24" }, { "dataPath": "params_shard_45.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.17.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.17.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.18.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.18.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.18.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.18.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.18.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.18.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.18.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "c90a16b3061ab7b096cc87e3233a261e" }, { "dataPath": "params_shard_46.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.19.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "e7d7724796ea027f21fc241fa950d82f" }, { "dataPath": "params_shard_47.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.18.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.18.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.19.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.19.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.19.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.19.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.19.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.19.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.19.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "e6296b8a38c460ef494d7b8fda7fc4ae" }, { "dataPath": "params_shard_48.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.2.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "9473ad9fec4bb2716b5f0c5b4dbbb733" }, { "dataPath": "params_shard_49.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.19.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.19.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.2.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.2.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.2.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.2.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.2.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.2.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.2.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "6f9b430931885d3fe3157f5cbcbead22" }, { "dataPath": "params_shard_50.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.20.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "fa0fb12a7527f282a4a88d643dd07762" }, { "dataPath": "params_shard_51.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.2.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.2.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.20.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14174208 }, { "name": "transformer.h.20.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 14180352 }, { "name": "transformer.h.20.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26763264 }, { "name": "transformer.h.20.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 26769408 }, { "name": "transformer.h.20.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26802176 }, { "name": "transformer.h.20.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 26808320 }, { "name": "transformer.h.20.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "5c42b53cbfebbabd3434870b1057ff2a" }, { "dataPath": "params_shard_52.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.3.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "224cc6ece2409cb0cd1db98ae8930d4c" }, { "dataPath": "params_shard_53.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.20.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.20.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 14155776 }, { "name": "transformer.h.21.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 14174208 }, { "name": "transformer.h.21.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18892800 }, { "name": "transformer.h.3.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18898944 }, { "name": "transformer.h.3.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 18905088 }, { "name": "transformer.h.3.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31488000 }, { "name": "transformer.h.3.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 31494144 }, { "name": "transformer.h.3.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "b8b37c0ceccada8afd8fa8129d14f691" }, { "dataPath": "params_shard_54.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.4.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "27ec5af382ecc3e83f5ebf56dd938165" }, { "dataPath": "params_shard_55.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.3.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.3.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "transformer.h.3.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 4724736 }, { "name": "transformer.h.3.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 18880512 }, { "name": "transformer.h.4.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18898944 }, { "name": "transformer.h.4.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 18905088 }, { "name": "transformer.h.4.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31488000 }, { "name": "transformer.h.4.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 31494144 }, { "name": "transformer.h.4.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "069b69e67d88c2f71ae72232c5d1470c" }, { "dataPath": "params_shard_56.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.5.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "74668c7c5adc5559d4b5de9d63be29a3" }, { "dataPath": "params_shard_57.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.4.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.4.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "transformer.h.4.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 4724736 }, { "name": "transformer.h.4.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 18880512 }, { "name": "transformer.h.5.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18898944 }, { "name": "transformer.h.5.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 18905088 }, { "name": "transformer.h.5.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31488000 }, { "name": "transformer.h.5.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 31494144 }, { "name": "transformer.h.5.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "dfdea0e2a073a8af2ae09c3df60db4b6" }, { "dataPath": "params_shard_58.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.6.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "dde8214e6f8bc953541b962841acfc10" }, { "dataPath": "params_shard_59.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.5.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.5.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "transformer.h.5.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 4724736 }, { "name": "transformer.h.5.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 18880512 }, { "name": "transformer.h.6.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18898944 }, { "name": "transformer.h.6.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 18905088 }, { "name": "transformer.h.6.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31488000 }, { "name": "transformer.h.6.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 31494144 }, { "name": "transformer.h.6.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "0e8bd021d72013ae474dec665bac29d3" }, { "dataPath": "params_shard_60.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.7.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "399d779dc25adb4f8d2f4a242a0325d8" }, { "dataPath": "params_shard_61.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.6.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.6.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "transformer.h.6.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 4724736 }, { "name": "transformer.h.6.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 18880512 }, { "name": "transformer.h.7.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18898944 }, { "name": "transformer.h.7.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 18905088 }, { "name": "transformer.h.7.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31488000 }, { "name": "transformer.h.7.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 31494144 }, { "name": "transformer.h.7.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "7bd42dfdf7beaa4b75fea38ee4db58aa" }, { "dataPath": "params_shard_62.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.8.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "84e51b96333101762768631d45831e4a" }, { "dataPath": "params_shard_63.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.7.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.7.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "transformer.h.7.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 4724736 }, { "name": "transformer.h.7.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 18880512 }, { "name": "transformer.h.8.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18898944 }, { "name": "transformer.h.8.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 18905088 }, { "name": "transformer.h.8.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31488000 }, { "name": "transformer.h.8.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 31494144 }, { "name": "transformer.h.8.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "a007cda96e4bdc08a1c6554265805f02" }, { "dataPath": "params_shard_64.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.9.mlp.gate_up_proj.q_weight", "shape": [ 3072, 8192 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "ad8a0ac022e52f685662cc9cddeb2cc2" }, { "dataPath": "params_shard_65.bin", "format": "raw-shard", "nbytes": 31533056, "records": [ { "name": "transformer.h.8.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.8.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "transformer.h.8.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 4724736 }, { "name": "transformer.h.8.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 18880512 }, { "name": "transformer.h.9.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18898944 }, { "name": "transformer.h.9.mlp.down_proj.q_weight", "shape": [ 8192, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 18905088 }, { "name": "transformer.h.9.mlp.down_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31488000 }, { "name": "transformer.h.9.mlp.gate_up_proj.q_scale", "shape": [ 1, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768, "byteOffset": 31494144 }, { "name": "transformer.h.9.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31526912 } ], "md5sum": "0b62aee7f19f72bb17186ec063fbcd51" }, { "dataPath": "params_shard_66.bin", "format": "raw-shard", "nbytes": 18898944, "records": [ { "name": "transformer.h.9.mixer.out_proj.q_weight", "shape": [ 3072, 1536 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.9.mixer.out_proj.q_scale", "shape": [ 1, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "transformer.h.9.mixer.qkv_proj.q_weight", "shape": [ 3072, 4608 ], "dtype": "int8", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 4724736 }, { "name": "transformer.h.9.mixer.qkv_proj.q_scale", "shape": [ 1, 9216 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 18432, "byteOffset": 18880512 } ], "md5sum": "3e6bdfe293248f2b2c94459b36938522" } ] }