Commit
•
6a41779
1
Parent(s):
51918b8
Update tokenizer_config.json (#4)
Browse files- Update tokenizer_config.json (355444241e9125f71194e7938c5023de86986e96)
Co-authored-by: Andrew Wang <[email protected]>
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
@@ -8027,7 +8027,7 @@
|
|
8027 |
},
|
8028 |
"bos_token": "<s>",
|
8029 |
"clean_up_tokenization_spaces": true,
|
8030 |
-
"chat_template": "{{'<extra_id_0>System'}}{% for message in messages %}{% if message['role'] == 'system' %}{{'\n' + message['content'].strip()}}{% if tools or contexts %}{{'\n'}}{% endif %}{% endif %}{% endfor %}{% if tools %}{% for tool in tools %}{{ '\n<tool> ' + tool|tojson + ' </tool>' }}{% endfor %}{% endif %}{% if contexts %}{% if tools %}{{'\n'}}{% endif %}{% for context in contexts %}{{ '\n<context> ' + context.strip() + ' </context>' }}{% endfor %}{% endif %}{{'\n\n'}}{% for message in messages %}{% if message['role'] == 'user' %}{{ '<extra_id_1>User\n' + message['content'].strip() + '\n' }}{% elif message['role'] == 'assistant' %}{{ '<extra_id_1>Assistant\n' + message['content'].strip() + '\n' }}{% elif message['role'] == 'tool' %}{{ '<extra_id_1>Tool\n' + message['content'].strip() + '\n' }}{% endif %}{% endfor %}{{'<extra_id_1>Assistant\n'}}",
|
8031 |
"eos_token": "</s>",
|
8032 |
"model_input_names": [
|
8033 |
"input_ids",
|
|
|
8027 |
},
|
8028 |
"bos_token": "<s>",
|
8029 |
"clean_up_tokenization_spaces": true,
|
8030 |
+
"chat_template": "{{'<extra_id_0>System'}}{% for message in messages %}{% if message['role'] == 'system' %}{{'\n' + message['content'].strip()}}{% if tools or contexts %}{{'\n'}}{% endif %}{% endif %}{% endfor %}{% if tools %}{% for tool in tools %}{{ '\n<tool> ' + tool|tojson + ' </tool>' }}{% endfor %}{% endif %}{% if contexts %}{% if tools %}{{'\n'}}{% endif %}{% for context in contexts %}{{ '\n<context> ' + context.strip() + ' </context>' }}{% endfor %}{% endif %}{{'\n\n'}}{% for message in messages %}{% if message['role'] == 'user' %}{{ '<extra_id_1>User\n' + message['content'].strip() + '\n' }}{% elif message['role'] == 'assistant' %}{{ '<extra_id_1>Assistant\n' + message['content'].strip() + '\n' }}{% elif message['role'] == 'tool' %}{{ '<extra_id_1>Tool\n' + message['content'].strip() + '\n' }}{% endif %}{% endfor %}{%- if add_generation_prompt %}{{'<extra_id_1>Assistant\n'}}{%- endif %}",
|
8031 |
"eos_token": "</s>",
|
8032 |
"model_input_names": [
|
8033 |
"input_ids",
|