Upload 5 files

added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "": 50270,
+  "<INGR_END>": 50264,
+  "<INGR_START>": 50262,
+  "<INPUT_END>": 50268,
+  "<INPUT_START>": 50267,
+  "<INSTR_END>": 50261,
+  "<INSTR_START>": 50259,
+  "<NEXT_INGR>": 50263,
+  "<NEXT_INPUT>": 50269,
+  "<NEXT_INSTR>": 50260,
+  "<RECIPE_END>": 50266,
+  "<RECIPE_START>": 50265,
+  "<TITLE_END>": 50258,
+  "<TITLE_START>": 50257
+}

special_tokens_map.json

@@ -0,0 +1,18 @@
+{
+  "additional_special_tokens": [
+    "<TITLE_START>",
+    "<TITLE_END>",
+    "<INSTR_START>",
+    "<NEXT_INSTR>",
+    "<INSTR_END>",
+    "<INGR_START>",
+    "<NEXT_INGR>",
+    "<INGR_END>",
+    "<RECIPE_START>",
+    "<RECIPE_END>",
+    "<INPUT_START>",
+    "<INPUT_END>",
+    "<NEXT_INPUT>"
+  ],
+  "pad_token": "<RECIPE_END>"
+}

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}