Upload tokenizer
#3
by
cat-state
- opened
- tokenizer.json +27 -0
tokenizer.json
CHANGED
|
@@ -227,6 +227,33 @@
|
|
| 227 |
"rstrip": false,
|
| 228 |
"normalized": true,
|
| 229 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
}
|
| 231 |
],
|
| 232 |
"normalizer": {
|
|
|
|
| 227 |
"rstrip": false,
|
| 228 |
"normalized": true,
|
| 229 |
"special": false
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"id": 50277,
|
| 233 |
+
"content": "<|SYSTEM|>",
|
| 234 |
+
"single_word": false,
|
| 235 |
+
"lstrip": false,
|
| 236 |
+
"rstrip": false,
|
| 237 |
+
"normalized": false,
|
| 238 |
+
"special": true
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"id": 50278,
|
| 242 |
+
"content": "<|USER|>",
|
| 243 |
+
"single_word": false,
|
| 244 |
+
"lstrip": false,
|
| 245 |
+
"rstrip": false,
|
| 246 |
+
"normalized": false,
|
| 247 |
+
"special": true
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"id": 50279,
|
| 251 |
+
"content": "<|ASSISTANT|>",
|
| 252 |
+
"single_word": false,
|
| 253 |
+
"lstrip": false,
|
| 254 |
+
"rstrip": false,
|
| 255 |
+
"normalized": false,
|
| 256 |
+
"special": true
|
| 257 |
}
|
| 258 |
],
|
| 259 |
"normalizer": {
|