jeradf commited on
Commit
464f8ab
1 Parent(s): ffac5c6

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +11 -29
special_tokens_map.json CHANGED
@@ -1,33 +1,9 @@
1
  {
2
  "additional_special_tokens": [
3
- {
4
- "content": "<|im_start|>",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false
9
- },
10
- {
11
- "content": "<|im_end|>",
12
- "lstrip": false,
13
- "normalized": false,
14
- "rstrip": false,
15
- "single_word": false
16
- },
17
- {
18
- "content": "<|user|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- },
24
- {
25
- "content": "<|assistant|>",
26
- "lstrip": false,
27
- "normalized": false,
28
- "rstrip": false,
29
- "single_word": false
30
- }
31
  ],
32
  "bos_token": {
33
  "content": "<|endoftext|>",
@@ -43,7 +19,13 @@
43
  "rstrip": false,
44
  "single_word": false
45
  },
46
- "pad_token": "<|endoftext|>",
 
 
 
 
 
 
47
  "unk_token": {
48
  "content": "<|endoftext|>",
49
  "lstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|user|>",
6
+ "<|assistant|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  ],
8
  "bos_token": {
9
  "content": "<|endoftext|>",
 
19
  "rstrip": false,
20
  "single_word": false
21
  },
22
+ "pad_token": {
23
+ "content": "<|endoftext|>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false
28
+ },
29
  "unk_token": {
30
  "content": "<|endoftext|>",
31
  "lstrip": false,