Training in progress, epoch 0

adapter_config.json

@@ -12,14 +12,14 @@
   "lora_dropout": 0.05,
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 64,
+  "r": 32,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "dense",
-    "query_key_value",
     "dense_h_to_4h",
-    "dense_4h_to_h"
+    "query_key_value",
+    "dense_4h_to_h",
+    "dense"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3b794f975f5e8d0346ee9b42adf3bd4dfd2aa206dd730bff8be88c0041ddf14
-size 522284877
+oid sha256:9b590af98d38b0cba5ffca897f94ff33198d51e1b6e0a1b94aba0f00222c1890
+size 261189453

config.json

@@ -21,6 +21,7 @@
   "hidden_size": 4544,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 2048,
   "model_type": "falcon",
   "multi_query": true,
   "new_decoder_architecture": false,
@@ -40,8 +41,10 @@
     "load_in_8bit": false,
     "quant_method": "bitsandbytes"
   },
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.34.0.dev0",
+  "transformers_version": "4.35.0.dev0",
   "use_cache": false,
-  "vocab_size": 65024
+  "vocab_size": 65027
 }

special_tokens_map.json

@@ -12,7 +12,8 @@
     ">>SUFFIX<<",
     ">>MIDDLE<<"
   ],
-  "bos_token": ">>ABSTRACT<<",
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>"
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
 }

tokenizer.json

@@ -7,8 +7,8 @@
       "id": 0,
       "content": ">>TITLE<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -16,8 +16,8 @@
       "id": 1,
       "content": ">>ABSTRACT<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -25,8 +25,8 @@
       "id": 2,
       "content": ">>INTRODUCTION<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -34,8 +34,8 @@
       "id": 3,
       "content": ">>SUMMARY<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -43,8 +43,8 @@
       "id": 4,
       "content": ">>COMMENT<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -52,8 +52,8 @@
       "id": 5,
       "content": ">>ANSWER<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -61,8 +61,8 @@
       "id": 6,
       "content": ">>QUESTION<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -70,8 +70,8 @@
       "id": 7,
       "content": ">>DOMAIN<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -79,8 +79,8 @@
       "id": 8,
       "content": ">>PREFIX<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -88,8 +88,8 @@
       "id": 9,
       "content": ">>SUFFIX<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -97,8 +97,8 @@
       "id": 10,
       "content": ">>MIDDLE<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -106,6 +106,33 @@
       "id": 11,
       "content": "<|endoftext|>",
       "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65024,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65025,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65026,
+      "content": "<unk>",
+      "single_word": false,
       "lstrip": true,
       "rstrip": true,
       "normalized": false,

tokenizer_config.json

@@ -3,94 +3,118 @@
   "added_tokens_decoder": {
     "0": {
       "content": ">>TITLE<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "1": {
       "content": ">>ABSTRACT<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "2": {
       "content": ">>INTRODUCTION<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "3": {
       "content": ">>SUMMARY<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "4": {
       "content": ">>COMMENT<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "5": {
       "content": ">>ANSWER<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "6": {
       "content": ">>QUESTION<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "7": {
       "content": ">>DOMAIN<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "8": {
       "content": ">>PREFIX<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "9": {
       "content": ">>SUFFIX<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "10": {
       "content": ">>MIDDLE<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "11": {
       "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65024": {
+      "content": "<s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "65025": {
+      "content": "</s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "65026": {
+      "content": "<unk>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -112,7 +136,7 @@
     ">>MIDDLE<<"
   ],
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "</s>",
   "model_input_names": [
     "input_ids",
     "attention_mask"

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80d4c6c13aa424e69e699ca029f25025f3867fadd4c6058fa91d42435b9a0b84
-size 4475
+oid sha256:72f4ab0bc1682946ae88003cc1cf4c559eca058b2c8743584d85f016de994137
+size 4539