| { | |
| "metadata": { | |
| "ParamSize": 325, | |
| "ParamBytes": 1925183488.0, | |
| "BitsPerParam": 4.0306587953497806 | |
| }, | |
| "records": [ | |
| { | |
| "dataPath": "params_shard_0.bin", | |
| "format": "raw-shard", | |
| "nbytes": 49250304, | |
| "records": [ | |
| { | |
| "name": "lm_head.q_weight", | |
| "shape": [ | |
| 32064, | |
| 384 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 49250304, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "5d3ade109e743cd5c15798661e7d8286" | |
| }, | |
| { | |
| "dataPath": "params_shard_1.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.21.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "78a4a5b20f833cb7651edbe81a1a0d5e" | |
| }, | |
| { | |
| "dataPath": "params_shard_2.bin", | |
| "format": "raw-shard", | |
| "nbytes": 32970752, | |
| "records": [ | |
| { | |
| "name": "lm_head.q_scale", | |
| "shape": [ | |
| 32064, | |
| 96 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6156288, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.21.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 6156288 | |
| }, | |
| { | |
| "name": "transformer.h.21.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 6162432 | |
| }, | |
| { | |
| "name": "transformer.h.21.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18745344 | |
| }, | |
| { | |
| "name": "transformer.h.21.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 18751488 | |
| }, | |
| { | |
| "name": "transformer.h.21.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18784256 | |
| }, | |
| { | |
| "name": "transformer.h.21.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 18790400 | |
| }, | |
| { | |
| "name": "transformer.h.21.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 32946176 | |
| }, | |
| { | |
| "name": "transformer.h.22.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 32964608 | |
| } | |
| ], | |
| "md5sum": "ec22f38e4e2eb3adcab232d2eefb787a" | |
| }, | |
| { | |
| "dataPath": "params_shard_3.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.22.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "1d6767246ead182f0a023599dc4f3c81" | |
| }, | |
| { | |
| "dataPath": "params_shard_4.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.22.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.22.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.22.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.22.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.22.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.22.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.22.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.22.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.23.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "a51cfe876723944c76711c90512ea8bd" | |
| }, | |
| { | |
| "dataPath": "params_shard_5.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.23.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "1cb656f01b57771845f9d0933cfc6759" | |
| }, | |
| { | |
| "dataPath": "params_shard_6.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.23.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.23.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.23.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.23.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.23.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.23.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.23.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.23.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.24.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "a3a1d182458f05aa629cdb7b6f345350" | |
| }, | |
| { | |
| "dataPath": "params_shard_7.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.24.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "992765155769413002936cf7a9ea4442" | |
| }, | |
| { | |
| "dataPath": "params_shard_8.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.24.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.24.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.24.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.24.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.24.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.24.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.24.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.24.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.25.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "b4f4b51b1bf9b62d435adf8315381b76" | |
| }, | |
| { | |
| "dataPath": "params_shard_9.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.25.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "9f9adfa4f11cd1d1d0752107f1f0599c" | |
| }, | |
| { | |
| "dataPath": "params_shard_10.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.25.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.25.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.25.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.25.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.25.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.25.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.25.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.25.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.26.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "b5f5e995de08a27639f925ec9b8190e4" | |
| }, | |
| { | |
| "dataPath": "params_shard_11.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.26.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "12fc1ea95688f5cae1cf9aac3deb7be4" | |
| }, | |
| { | |
| "dataPath": "params_shard_12.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.26.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.26.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.26.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.26.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.26.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.26.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.26.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.26.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.27.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "707c78d752687048395d4ab2f28ef091" | |
| }, | |
| { | |
| "dataPath": "params_shard_13.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.27.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "9781ce88fb38c7010bfcf4fbfdf790c3" | |
| }, | |
| { | |
| "dataPath": "params_shard_14.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.27.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.27.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.27.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.27.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.27.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.27.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.27.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.27.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.28.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "be7dabb40a3101ca81294834939c2a4f" | |
| }, | |
| { | |
| "dataPath": "params_shard_15.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.28.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "2bb1fb07b2b23de1b3a2d154082f5563" | |
| }, | |
| { | |
| "dataPath": "params_shard_16.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.28.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.28.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.28.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.28.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.28.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.28.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.28.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.28.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.29.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "570d9a59ec7712508eec93aa0bacdf8e" | |
| }, | |
| { | |
| "dataPath": "params_shard_17.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.29.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "d4ed37b61694548dbfab578a073d7350" | |
| }, | |
| { | |
| "dataPath": "params_shard_18.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.29.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.29.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.29.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.29.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.29.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.29.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.29.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.29.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.30.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "a0b172795ba6f3df91eff773779c5e09" | |
| }, | |
| { | |
| "dataPath": "params_shard_19.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.30.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "5351f3145599b8e12503deee977d6939" | |
| }, | |
| { | |
| "dataPath": "params_shard_20.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.30.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.30.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.30.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.30.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.30.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.30.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.30.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.30.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.h.31.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "c31d1ae59506dc162a160ef4db715395" | |
| }, | |
| { | |
| "dataPath": "params_shard_21.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.31.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "28488ad3fdde10a98e057d9df8ab6916" | |
| }, | |
| { | |
| "dataPath": "params_shard_22.bin", | |
| "format": "raw-shard", | |
| "nbytes": 49250304, | |
| "records": [ | |
| { | |
| "name": "transformer.embd.q_weight", | |
| "shape": [ | |
| 32064, | |
| 384 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 49250304, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3f9a3cb77fdaedfd62e9720fc14e716e" | |
| }, | |
| { | |
| "dataPath": "params_shard_23.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.31.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.31.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12582912 | |
| }, | |
| { | |
| "name": "transformer.h.31.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 12589056 | |
| }, | |
| { | |
| "name": "transformer.h.31.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 12621824 | |
| }, | |
| { | |
| "name": "transformer.h.31.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 12627968 | |
| }, | |
| { | |
| "name": "transformer.h.31.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 17346560 | |
| }, | |
| { | |
| "name": "transformer.h.31.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 17352704 | |
| }, | |
| { | |
| "name": "transformer.h.31.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 31508480 | |
| }, | |
| { | |
| "name": "transformer.norm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "a82cc1c4e8a3a7ebb02d781ae768de39" | |
| }, | |
| { | |
| "dataPath": "params_shard_24.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "10d41991771e6254005c82a8e3d55795" | |
| }, | |
| { | |
| "dataPath": "params_shard_25.bin", | |
| "format": "raw-shard", | |
| "nbytes": 23515136, | |
| "records": [ | |
| { | |
| "name": "transformer.embd.q_scale", | |
| "shape": [ | |
| 32064, | |
| 96 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6156288, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.0.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 6156288 | |
| }, | |
| { | |
| "name": "transformer.h.0.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 6162432 | |
| }, | |
| { | |
| "name": "transformer.h.0.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18745344 | |
| }, | |
| { | |
| "name": "transformer.h.0.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 18751488 | |
| }, | |
| { | |
| "name": "transformer.h.0.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18784256 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 18790400 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 23508992 | |
| } | |
| ], | |
| "md5sum": "bf54fa3e7550c7f42036f02f0626d222" | |
| }, | |
| { | |
| "dataPath": "params_shard_26.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8a25aac59c6eeb5df903f883f7a5d540" | |
| }, | |
| { | |
| "dataPath": "params_shard_27.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.1.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.1.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.1.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.1.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.1.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "9b4a683770c286c3e7d5e9120ef1d5dd" | |
| }, | |
| { | |
| "dataPath": "params_shard_28.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "c34505f475449bb2c465b0a633a25793" | |
| }, | |
| { | |
| "dataPath": "params_shard_29.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.10.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.10.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.10.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.10.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.10.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.10.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.10.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "76885693cac42f873bb691ea6fb57b0c" | |
| }, | |
| { | |
| "dataPath": "params_shard_30.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "7023c08ce4586f7fc873bbc6d1149d8e" | |
| }, | |
| { | |
| "dataPath": "params_shard_31.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.10.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.11.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.11.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.11.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.11.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.11.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "ba20f39eec0167be058927c1e5aae82a" | |
| }, | |
| { | |
| "dataPath": "params_shard_32.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "f9f2bf14df09314718d370f0e86f97b0" | |
| }, | |
| { | |
| "dataPath": "params_shard_33.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.12.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.12.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.12.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.12.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.12.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.12.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.12.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "6dad8994e7265368eb895c8dba21715b" | |
| }, | |
| { | |
| "dataPath": "params_shard_34.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "30dda05e0880bdb98b4067d5f90baa8e" | |
| }, | |
| { | |
| "dataPath": "params_shard_35.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.12.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.13.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.13.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.13.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.13.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.13.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "52bf5c91439431490af7a116788bea08" | |
| }, | |
| { | |
| "dataPath": "params_shard_36.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "f8e804a983415caca4e31df3047a6e1a" | |
| }, | |
| { | |
| "dataPath": "params_shard_37.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.14.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.14.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.14.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.14.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.14.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.14.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.14.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "8d1484cbe9cad4e0c1e211b4af7b07ad" | |
| }, | |
| { | |
| "dataPath": "params_shard_38.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "09aa9091a48044a5f6e769ff74b9c8fa" | |
| }, | |
| { | |
| "dataPath": "params_shard_39.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.14.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.15.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.15.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.15.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.15.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.15.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "8dd2efe06dcb3ee1958064aa0f8d7dcc" | |
| }, | |
| { | |
| "dataPath": "params_shard_40.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "f9e0d79d96e92283a0cc0f4c8db060e1" | |
| }, | |
| { | |
| "dataPath": "params_shard_41.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.16.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.16.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.16.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.16.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.16.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.16.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.16.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "91c16d869e8b43e56fe011ddefdaca1d" | |
| }, | |
| { | |
| "dataPath": "params_shard_42.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "e9a5dd6b887e8aae0c5e162438da9365" | |
| }, | |
| { | |
| "dataPath": "params_shard_43.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.16.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.17.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.17.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.17.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.17.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.17.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "8784e6830067f7ca05e331976b50a12d" | |
| }, | |
| { | |
| "dataPath": "params_shard_44.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "102f3b65b0faa129450be3a420f43d24" | |
| }, | |
| { | |
| "dataPath": "params_shard_45.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.18.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.18.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.18.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.18.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.18.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.18.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.18.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "c90a16b3061ab7b096cc87e3233a261e" | |
| }, | |
| { | |
| "dataPath": "params_shard_46.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "e7d7724796ea027f21fc241fa950d82f" | |
| }, | |
| { | |
| "dataPath": "params_shard_47.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.18.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.19.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.19.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.19.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.19.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.19.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "e6296b8a38c460ef494d7b8fda7fc4ae" | |
| }, | |
| { | |
| "dataPath": "params_shard_48.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "9473ad9fec4bb2716b5f0c5b4dbbb733" | |
| }, | |
| { | |
| "dataPath": "params_shard_49.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.2.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.2.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.2.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.2.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.2.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.2.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.2.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "6f9b430931885d3fe3157f5cbcbead22" | |
| }, | |
| { | |
| "dataPath": "params_shard_50.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.20.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "fa0fb12a7527f282a4a88d643dd07762" | |
| }, | |
| { | |
| "dataPath": "params_shard_51.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.2.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.20.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.20.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 14180352 | |
| }, | |
| { | |
| "name": "transformer.h.20.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26763264 | |
| }, | |
| { | |
| "name": "transformer.h.20.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 26769408 | |
| }, | |
| { | |
| "name": "transformer.h.20.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 26802176 | |
| }, | |
| { | |
| "name": "transformer.h.20.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 26808320 | |
| }, | |
| { | |
| "name": "transformer.h.20.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "5c42b53cbfebbabd3434870b1057ff2a" | |
| }, | |
| { | |
| "dataPath": "params_shard_52.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "224cc6ece2409cb0cd1db98ae8930d4c" | |
| }, | |
| { | |
| "dataPath": "params_shard_53.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.20.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.20.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 14155776 | |
| }, | |
| { | |
| "name": "transformer.h.21.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 14174208 | |
| }, | |
| { | |
| "name": "transformer.h.21.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18892800 | |
| }, | |
| { | |
| "name": "transformer.h.3.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18898944 | |
| }, | |
| { | |
| "name": "transformer.h.3.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 18905088 | |
| }, | |
| { | |
| "name": "transformer.h.3.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31488000 | |
| }, | |
| { | |
| "name": "transformer.h.3.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 31494144 | |
| }, | |
| { | |
| "name": "transformer.h.3.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "b8b37c0ceccada8afd8fa8129d14f691" | |
| }, | |
| { | |
| "dataPath": "params_shard_54.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "27ec5af382ecc3e83f5ebf56dd938165" | |
| }, | |
| { | |
| "dataPath": "params_shard_55.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 4724736 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 18880512 | |
| }, | |
| { | |
| "name": "transformer.h.4.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18898944 | |
| }, | |
| { | |
| "name": "transformer.h.4.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 18905088 | |
| }, | |
| { | |
| "name": "transformer.h.4.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31488000 | |
| }, | |
| { | |
| "name": "transformer.h.4.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 31494144 | |
| }, | |
| { | |
| "name": "transformer.h.4.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "069b69e67d88c2f71ae72232c5d1470c" | |
| }, | |
| { | |
| "dataPath": "params_shard_56.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "74668c7c5adc5559d4b5de9d63be29a3" | |
| }, | |
| { | |
| "dataPath": "params_shard_57.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 4724736 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 18880512 | |
| }, | |
| { | |
| "name": "transformer.h.5.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18898944 | |
| }, | |
| { | |
| "name": "transformer.h.5.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 18905088 | |
| }, | |
| { | |
| "name": "transformer.h.5.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31488000 | |
| }, | |
| { | |
| "name": "transformer.h.5.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 31494144 | |
| }, | |
| { | |
| "name": "transformer.h.5.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "dfdea0e2a073a8af2ae09c3df60db4b6" | |
| }, | |
| { | |
| "dataPath": "params_shard_58.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "dde8214e6f8bc953541b962841acfc10" | |
| }, | |
| { | |
| "dataPath": "params_shard_59.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 4724736 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 18880512 | |
| }, | |
| { | |
| "name": "transformer.h.6.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18898944 | |
| }, | |
| { | |
| "name": "transformer.h.6.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 18905088 | |
| }, | |
| { | |
| "name": "transformer.h.6.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31488000 | |
| }, | |
| { | |
| "name": "transformer.h.6.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 31494144 | |
| }, | |
| { | |
| "name": "transformer.h.6.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "0e8bd021d72013ae474dec665bac29d3" | |
| }, | |
| { | |
| "dataPath": "params_shard_60.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "399d779dc25adb4f8d2f4a242a0325d8" | |
| }, | |
| { | |
| "dataPath": "params_shard_61.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 4724736 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 18880512 | |
| }, | |
| { | |
| "name": "transformer.h.7.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18898944 | |
| }, | |
| { | |
| "name": "transformer.h.7.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 18905088 | |
| }, | |
| { | |
| "name": "transformer.h.7.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31488000 | |
| }, | |
| { | |
| "name": "transformer.h.7.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 31494144 | |
| }, | |
| { | |
| "name": "transformer.h.7.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "7bd42dfdf7beaa4b75fea38ee4db58aa" | |
| }, | |
| { | |
| "dataPath": "params_shard_62.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "84e51b96333101762768631d45831e4a" | |
| }, | |
| { | |
| "dataPath": "params_shard_63.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 4724736 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 18880512 | |
| }, | |
| { | |
| "name": "transformer.h.8.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18898944 | |
| }, | |
| { | |
| "name": "transformer.h.8.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 18905088 | |
| }, | |
| { | |
| "name": "transformer.h.8.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31488000 | |
| }, | |
| { | |
| "name": "transformer.h.8.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 31494144 | |
| }, | |
| { | |
| "name": "transformer.h.8.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "a007cda96e4bdc08a1c6554265805f02" | |
| }, | |
| { | |
| "dataPath": "params_shard_64.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25165824, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mlp.gate_up_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 8192 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 25165824, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "ad8a0ac022e52f685662cc9cddeb2cc2" | |
| }, | |
| { | |
| "dataPath": "params_shard_65.bin", | |
| "format": "raw-shard", | |
| "nbytes": 31533056, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 4724736 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 18880512 | |
| }, | |
| { | |
| "name": "transformer.h.9.ln.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 18898944 | |
| }, | |
| { | |
| "name": "transformer.h.9.mlp.down_proj.q_weight", | |
| "shape": [ | |
| 8192, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 12582912, | |
| "byteOffset": 18905088 | |
| }, | |
| { | |
| "name": "transformer.h.9.mlp.down_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31488000 | |
| }, | |
| { | |
| "name": "transformer.h.9.mlp.gate_up_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 16384 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 32768, | |
| "byteOffset": 31494144 | |
| }, | |
| { | |
| "name": "transformer.h.9.post_attention_layernorm.weight", | |
| "shape": [ | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 31526912 | |
| } | |
| ], | |
| "md5sum": "0b62aee7f19f72bb17186ec063fbcd51" | |
| }, | |
| { | |
| "dataPath": "params_shard_66.bin", | |
| "format": "raw-shard", | |
| "nbytes": 18898944, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mixer.out_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 1536 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4718592, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.out_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 3072 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 6144, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.qkv_proj.q_weight", | |
| "shape": [ | |
| 3072, | |
| 4608 | |
| ], | |
| "dtype": "int8", | |
| "format": "f32-to-bf16", | |
| "nbytes": 14155776, | |
| "byteOffset": 4724736 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.qkv_proj.q_scale", | |
| "shape": [ | |
| 1, | |
| 9216 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 18432, | |
| "byteOffset": 18880512 | |
| } | |
| ], | |
| "md5sum": "3e6bdfe293248f2b2c94459b36938522" | |
| } | |
| ] | |
| } |