TrevorJS/mtg-phi-1_5-2-dpo

README.md

@@ -1,6 +1,6 @@
 ---
-license: llama2
-base_model: codellama/CodeLlama-7b-hf
+license: other
+base_model: microsoft/phi-1_5
 tags:
 - generated_from_trainer
 model-index:
@@ -13,17 +13,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # dpo
 
-This model is a fine-tuned version of [codellama/CodeLlama-7b-hf](https://huggingface.co/codellama/CodeLlama-7b-hf) on the None dataset.
+This model is a fine-tuned version of [microsoft/phi-1_5](https://huggingface.co/microsoft/phi-1_5) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0010
-- Rewards/chosen: -5.5160
-- Rewards/rejected: -16.3854
-- Rewards/accuracies: 0.9991
-- Rewards/margins: 10.8694
-- Logps/rejected: -189.5804
-- Logps/chosen: -75.4962
-- Logits/rejected: -1.9168
-- Logits/chosen: -2.2651
+- Loss: 0.0000
+- Rewards/chosen: -8.4849
+- Rewards/rejected: -25.9483
+- Rewards/accuracies: 1.0
+- Rewards/margins: 17.4633
+- Logps/rejected: -293.3352
+- Logps/chosen: -152.1862
+- Logits/rejected: -0.9014
+- Logits/chosen: -0.4994
 
 ## Model description
 
@@ -51,27 +51,42 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 100
-- training_steps: 1000
+- training_steps: 2500
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0937        | 0.07  | 100  | 0.1058          | -5.7722        | -13.1291         | 0.9515             | 7.3569          | -157.0180      | -78.0587     | -1.5497         | -1.7945       |
-| 0.0442        | 0.15  | 200  | 0.0392          | -4.5353        | -13.5199         | 0.9835             | 8.9845          | -160.9253      | -65.6901     | -2.3349         | -2.6709       |
-| 0.0165        | 0.22  | 300  | 0.0186          | -5.9014        | -14.9526         | 0.9939             | 9.0511          | -175.2523      | -79.3511     | -2.1283         | -2.4222       |
-| 0.0016        | 0.29  | 400  | 0.0239          | -7.2028        | -18.5000         | 0.9922             | 11.2973         | -210.7271      | -92.3644     | -1.9903         | -2.3400       |
-| 0.0014        | 0.36  | 500  | 0.0065          | -5.1522        | -15.5259         | 0.9983             | 10.3737         | -180.9857      | -71.8588     | -1.6803         | -2.0312       |
-| 0.0006        | 0.44  | 600  | 0.0038          | -5.0346        | -14.9248         | 0.9991             | 9.8902          | -174.9747      | -70.6829     | -2.1480         | -2.4897       |
-| 0.0003        | 0.51  | 700  | 0.0018          | -5.2732        | -16.3551         | 0.9991             | 11.0819         | -189.2777      | -73.0690     | -1.9657         | -2.3112       |
-| 0.0002        | 0.58  | 800  | 0.0016          | -5.5844        | -16.3258         | 1.0                | 10.7414         | -188.9845      | -76.1804     | -1.9284         | -2.2717       |
-| 0.0004        | 0.66  | 900  | 0.0011          | -5.5422        | -16.3921         | 0.9991             | 10.8499         | -189.6474      | -75.7582     | -1.9152         | -2.2631       |
-| 0.0008        | 0.73  | 1000 | 0.0010          | -5.5160        | -16.3854         | 0.9991             | 10.8694         | -189.5804      | -75.4962     | -1.9168         | -2.2651       |
+| 0.0318        | 0.07  | 100  | 0.0384          | -0.3956        | -7.7708          | 0.9835             | 7.3753          | -111.5607      | -71.2923     | 1.1941          | 1.0925        |
+| 0.0187        | 0.15  | 200  | 0.0196          | -2.0328        | -10.9862         | 0.9922             | 8.9535          | -143.7145      | -87.6645     | -0.8539         | -0.9067       |
+| 0.0101        | 0.22  | 300  | 0.0351          | -2.7345        | -12.1219         | 0.9896             | 9.3874          | -155.0717      | -94.6821     | 0.4420          | 0.5220        |
+| 0.046         | 0.29  | 400  | 0.0199          | -6.6027        | -18.5556         | 0.9922             | 11.9529         | -219.4086      | -133.3638    | -2.3908         | -2.0500       |
+| 0.0005        | 0.36  | 500  | 0.0101          | -6.4299        | -20.5496         | 0.9965             | 14.1197         | -239.3484      | -131.6356    | -1.0029         | -0.6334       |
+| 0.0003        | 0.44  | 600  | 0.0092          | -9.0181        | -23.0513         | 0.9965             | 14.0332         | -264.3652      | -157.5181    | -1.6334         | -1.1488       |
+| 0.0004        | 0.51  | 700  | 0.0043          | -5.7377        | -21.3127         | 0.9991             | 15.5749         | -246.9788      | -124.7142    | -0.8477         | -0.4037       |
+| 0.0001        | 0.58  | 800  | 0.0040          | -8.9021        | -23.9436         | 0.9991             | 15.0415         | -273.2885      | -156.3581    | 0.2782          | 0.8244        |
+| 0.0001        | 0.66  | 900  | 0.0031          | -9.3191        | -24.3563         | 0.9991             | 15.0371         | -277.4149      | -160.5282    | -0.7279         | -0.2168       |
+| 0.002         | 0.73  | 1000 | 0.0066          | -6.8680        | -23.5822         | 0.9974             | 16.7142         | -269.6745      | -136.0172    | -0.6629         | 0.2962        |
+| 0.0002        | 0.8   | 1100 | 0.0015          | -9.1417        | -27.6276         | 0.9991             | 18.4859         | -310.1280      | -158.7536    | -1.2030         | -0.5215       |
+| 0.0823        | 0.87  | 1200 | 0.0057          | -4.4568        | -18.4378         | 0.9974             | 13.9810         | -218.2306      | -111.9051    | 0.2236          | 0.7934        |
+| 0.0           | 0.95  | 1300 | 0.0171          | -8.1530        | -25.5603         | 0.9983             | 17.4073         | -289.4550      | -148.8665    | -1.2413         | -0.9611       |
+| 0.0007        | 1.02  | 1400 | 0.0019          | -7.9402        | -25.1905         | 0.9983             | 17.2503         | -285.7569      | -146.7384    | -1.2325         | -0.8924       |
+| 0.0002        | 1.09  | 1500 | 0.0010          | -8.1543        | -25.2960         | 0.9991             | 17.1417         | -286.8122      | -148.8794    | -1.0005         | -0.6261       |
+| 0.0           | 1.17  | 1600 | 0.0010          | -8.4019        | -25.6275         | 0.9991             | 17.2256         | -290.1275      | -151.3556    | -1.0850         | -0.7170       |
+| 0.0           | 1.24  | 1700 | 0.0011          | -8.8691        | -26.2284         | 0.9991             | 17.3593         | -296.1366      | -156.0278    | -1.1426         | -0.7830       |
+| 0.0           | 1.31  | 1800 | 0.0010          | -9.2896        | -26.9277         | 0.9991             | 17.6381         | -303.1297      | -160.2331    | -1.1169         | -0.7512       |
+| 0.0001        | 1.39  | 1900 | 0.0011          | -9.2869        | -26.9301         | 0.9991             | 17.6432         | -303.1532      | -160.2053    | -1.1213         | -0.7560       |
+| 0.0           | 1.46  | 2000 | 0.0008          | -8.4453        | -25.9094         | 0.9991             | 17.4641         | -292.9459      | -151.7894    | -0.8854         | -0.4791       |
+| 0.0           | 1.53  | 2100 | 0.0007          | -8.4600        | -25.9284         | 0.9991             | 17.4684         | -293.1361      | -151.9364    | -0.8893         | -0.4835       |
+| 0.0           | 1.6   | 2200 | 0.0000          | -8.4501        | -25.9071         | 1.0                | 17.4569         | -292.9228      | -151.8381    | -0.8823         | -0.4759       |
+| 0.0           | 1.68  | 2300 | 0.0000          | -8.4800        | -25.9444         | 1.0                | 17.4644         | -293.2967      | -152.1372    | -0.8982         | -0.4964       |
+| 0.0           | 1.75  | 2400 | 0.0000          | -8.4864        | -25.9459         | 1.0                | 17.4596         | -293.3117      | -152.2005    | -0.9013         | -0.4999       |
+| 0.0           | 1.82  | 2500 | 0.0000          | -8.4849        | -25.9483         | 1.0                | 17.4633         | -293.3352      | -152.1862    | -0.9014         | -0.4994       |
 
 
 ### Framework versions
 
-- Transformers 4.33.1
+- Transformers 4.33.2
 - Pytorch 2.0.1+cu118
 - Datasets 2.14.5
 - Tokenizers 0.13.3

adapter_config.json

@@ -7,20 +7,31 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
-  "lora_alpha": 16,
+  "lora_alpha": 64,
   "lora_dropout": 0.05,
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 64,
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "v_proj",
-    "k_proj",
-    "o_proj",
-    "gate_proj",
-    "up_proj",
-    "down_proj"
+    "layers.0.wte",
+    "layers.20.mixer.Wqkv",
+    "layers.21.mixer.Wqkv",
+    "layers.22.mixer.Wqkv",
+    "layers.23.mixer.Wqkv",
+    "layers.24.mixer.Wqkv",
+    "layers.20.mixer.out_proj",
+    "layers.21.mixer.out_proj",
+    "layers.22.mixer.out_proj",
+    "layers.23.mixer.out_proj",
+    "layers.24.mixer.out_proj",
+    "layers.11.mlp.fc1",
+    "layers.17.mlp.fc1",
+    "layers.24.mlp.fc1",
+    "layers.11.mlp.fc2",
+    "layers.17.mlp.fc2",
+    "layers.24.mlp.fc2",
+    "layers.25.linear"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87ea7e6e6940a3bffa6ba4ec1c1a45920c5cee96163f0ebf2df01b1b3bc9a6ab
-size 80122381
+oid sha256:24650f2e31518c8264b666c94a4ceedc45bcfdf7e1cc9a75109160b5cf4b56e9
+size 29373021

added_tokens.json

@@ -0,0 +1,40 @@
+{
+  "\t\t": 50294,
+  "\t\t\t": 50293,
+  "\t\t\t\t": 50292,
+  "\t\t\t\t\t": 50291,
+  "\t\t\t\t\t\t": 50290,
+  "\t\t\t\t\t\t\t": 50289,
+  "\t\t\t\t\t\t\t\t": 50288,
+  "\t\t\t\t\t\t\t\t\t": 50287,
+  "  ": 50286,
+  "   ": 50285,
+  "    ": 50284,
+  "     ": 50283,
+  "      ": 50282,
+  "       ": 50281,
+  "        ": 50280,
+  "         ": 50279,
+  "          ": 50278,
+  "           ": 50277,
+  "            ": 50276,
+  "             ": 50275,
+  "              ": 50274,
+  "               ": 50273,
+  "                ": 50272,
+  "                 ": 50271,
+  "                  ": 50270,
+  "                   ": 50269,
+  "                    ": 50268,
+  "                     ": 50267,
+  "                      ": 50266,
+  "                       ": 50265,
+  "                        ": 50264,
+  "                         ": 50263,
+  "                          ": 50262,
+  "                           ": 50261,
+  "                            ": 50260,
+  "                             ": 50259,
+  "                              ": 50258,
+  "                               ": 50257
+}

special_tokens_map.json

@@ -1,30 +1,6 @@
 {
-  "additional_special_tokens": [
-    "▁<PRE>",
-    "▁<MID>",
-    "▁<SUF>",
-    "▁<EOT>"
-  ],
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "</s>",
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
 }

tokenizer_config.json

@@ -1,44 +1,9 @@
 {
-  "additional_special_tokens": [
-    "▁<PRE>",
-    "▁<MID>",
-    "▁<SUF>",
-    "▁<EOT>"
-  ],
-  "bos_token": {
-    "__type": "AddedToken",
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "clean_up_tokenization_spaces": false,
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eot_token": "▁<EOT>",
-  "fill_token": "<FILL_ME>",
-  "legacy": null,
-  "middle_token": "▁<MID>",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
-  "prefix_token": "▁<PRE>",
-  "sp_model_kwargs": {},
-  "suffix_token": "▁<SUF>",
-  "tokenizer_class": "CodeLlamaTokenizer",
-  "unk_token": {
-    "__type": "AddedToken",
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "use_default_system_prompt": false
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "tokenizer_class": "CodeGenTokenizer",
+  "unk_token": "<|endoftext|>"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0789fb5cf14abad8e57dc19f2dafc11a51d431e860bcf6d1215e733113e8c29
+oid sha256:515ba53b3d6464b6b6b3a1eac8d50fbe3b3fc34481b9a260b8edb4b52ba76de5
 size 4027