Upload ONNX weights + chat template fixes (#13)

- [WIP] Upload ONNX weights (c530c624be808758874da8b5890570cc3355c2c2)
- Update config.json (0f22893f43667f285e2d84805688e39960b35815)
- Upload folder using huggingface_hub (99e24a4e309be3d71aab5f6d986af677abc947a3)
- Update chat_template.json (748d43f20651a3ecb52a92af341fd5cc91538d90)
- Update tokenizer_config.json (b802c03d3ba20f3f84664dd9e08234b6d03650ec)
- Upload folder using huggingface_hub (91959f09ef43e11129cced282c37b5bebc85fb2c)
- Update config.json (5f8a748a46b8da8f1f29eaa49ff06ee50307befb)


Co-authored-by: Joshua <[email protected]>

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+onnx/decoder_model_merged.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/decoder_model_merged_fp16.onnx_data filter=lfs diff=lfs merge=lfs -text

chat_template.json

@@ -1,3 +1,3 @@
 {
-  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'].capitalize()}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"
+  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"
 }

config.json

@@ -170,6 +170,17 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.46.0",
+  "transformers.js_config": {
+    "kv_cache_dtype": {
+      "q4f16": "float16",
+      "fp16": "float16"
+    },
+    "dtype": {
+      "embed_tokens": "auto",
+      "vision_encoder": "auto",
+      "decoder_model_merged": "q4"
+    }
+  },
   "use_cache": true,
   "vision_config": {
     "size": {"longest_edge": 1920},

onnx/decoder_model_merged.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a27ef6fe177d3109e0913c63da7a4b0f2791fab95da3e5f91b31ba6e03115385
+size 126930

onnx/decoder_model_merged.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d530c318000311b2697d0b891ef46c69f9e9c89688761e043654d08a3cca376c
+size 6849724416

onnx/decoder_model_merged_bnb4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39974ffc8a05f4de601005dc555d326f3dd2744ffd544e2892ad065fe25b2b8a
+size 967330291

onnx/decoder_model_merged_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18906c3a7f15ae8176e0208184510ebacb8f65c66f7e80e89226918a267c147a
+size 1342510363

onnx/decoder_model_merged_fp16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4249b577dcd1cd146c6db45bea29108dd1e3831f1c6a5d6a226d01ac92ab411d
+size 2082471936

onnx/decoder_model_merged_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a2369559862dd3e40361a2b63ac0e7be18c07c72845393ee89df0e79713f6c7
+size 1716139218

onnx/decoder_model_merged_q4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:612e5c30793bc2045f9262b597013a25bcca44b4f76a7db196938a57a77e1f79
+size 1074284508

onnx/decoder_model_merged_quantized.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bddb1dcd933e681eb2a542186c081dd1e6cf4b67161d905ef9da31cabbd3474d
+size 1716139269

onnx/decoder_model_merged_uint8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bddb1dcd933e681eb2a542186c081dd1e6cf4b67161d905ef9da31cabbd3474d
+size 1716139269

onnx/embed_tokens.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ec8537866d20b78e618e15aea8f91a558266cd77fe783e513f095fc1de1c8c4
+size 402678062

onnx/embed_tokens_bnb4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eca0a3199567ba01a76dc6b923fd14bce39d6eb51d26686654bb7a98acfad280
+size 402678081

onnx/embed_tokens_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:377adedd6ac1975e3afc3fb4c24dd6032a973626da71a5e0648dec3735a56527
+size 201339266

onnx/embed_tokens_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6666a926ca2a65f89016ea19ec7c5b8afd01c58e5aca1f33733f2d936f31c71d
+size 100669984

onnx/embed_tokens_q4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eca0a3199567ba01a76dc6b923fd14bce39d6eb51d26686654bb7a98acfad280
+size 402678081

onnx/embed_tokens_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d835e5524c9a8b349fe55a7f589ab21780417c2f1e67f52062cf7787dcbefc3b
+size 201339285

onnx/embed_tokens_quantized.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6666a926ca2a65f89016ea19ec7c5b8afd01c58e5aca1f33733f2d936f31c71d
+size 100669984

onnx/embed_tokens_uint8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6666a926ca2a65f89016ea19ec7c5b8afd01c58e5aca1f33733f2d936f31c71d
+size 100669984

onnx/vision_encoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65bb9b57b64763897cc6dc397450449fce5607138843566a885e2f0a250343c8
+size 1737427560

onnx/vision_encoder_bnb4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a8256b74fd9465f859fab31c1840ed073aa0edd7b75d61127eefe1ce1fcf560
+size 251407732

onnx/vision_encoder_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab171611906fa056c91a28fde0ef1fda897b44bbf1ca0d9ae692cfaff90947b1
+size 868985807

onnx/vision_encoder_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:352fe86ad7d8358f39fb896de9b2efd0d8a6cf2b6239565841bab5146a735d2f
+size 436180765

onnx/vision_encoder_q4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d20e0f8a6201e4944759f2fbcab4fa035bbb1fb34e14700f25f1f00e678992
+size 278736452

onnx/vision_encoder_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b47406ea04d0754ccdd5bd0d68e827a72979f962886cf9bdeae926342234298
+size 247852840

onnx/vision_encoder_quantized.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e7292275057cec773aad0218310041e325289d18b708f89deae541925f4274
+size 436180848

onnx/vision_encoder_uint8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e7292275057cec773aad0218310041e325289d18b708f89deae541925f4274
+size 436180848

tokenizer_config.json

@@ -176,7 +176,7 @@
   "processor_class": "Idefics3Processor",
   "tokenizer_class": "GPT2Tokenizer",
   "truncation_side": "left",
-  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'].capitalize()}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}",
+  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}",
   "unk_token": "<|endoftext|>",
   "vocab_size": 49152
 }