{
  "best_metric": 0.31609994173049927,
  "best_model_checkpoint": "saves/LLaMA2-7B-Chat/lora/2023-08-25-11-47-37/checkpoint-500",
  "epoch": 0.8565310492505354,
  "global_step": 600,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 4.9999300626531746e-05,
      "logits/chosen": -0.7403622269630432,
      "logits/rejected": -0.6866486668586731,
      "logps/chosen": -181.39700317382812,
      "logps/rejected": -208.150146484375,
      "loss": 0.6892,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -5.543716179090552e-05,
      "rewards/margins": 0.008117685094475746,
      "rewards/rejected": -0.008173122070729733,
      "step": 5
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.999720254525684e-05,
      "logits/chosen": -0.7327243089675903,
      "logits/rejected": -0.6666526794433594,
      "logps/chosen": -231.14956665039062,
      "logps/rejected": -259.27978515625,
      "loss": 0.6702,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.010688358917832375,
      "rewards/margins": 0.04831721633672714,
      "rewards/rejected": -0.05900556966662407,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 4.9995272362322506e-05,
      "logits/chosen": -0.7042385935783386,
      "logits/rejected": -0.6349071264266968,
      "logps/chosen": -188.58367919921875,
      "logps/rejected": -223.201416015625,
      "loss": 0.6677,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07146739959716797,
      "rewards/margins": 0.05914110690355301,
      "rewards/rejected": -0.13060849905014038,
      "step": 15
    },
    {
      "epoch": 0.03,
      "learning_rate": 4.999093662527553e-05,
      "logits/chosen": -0.6996177434921265,
      "logits/rejected": -0.6077739000320435,
      "logps/chosen": -196.89700317382812,
      "logps/rejected": -243.52450561523438,
      "loss": 0.6027,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0204925499856472,
      "rewards/margins": 0.22103042900562286,
      "rewards/rejected": -0.24152295291423798,
      "step": 20
    },
    {
      "epoch": 0.04,
      "learning_rate": 4.998520264838675e-05,
      "logits/chosen": -0.6775780916213989,
      "logits/rejected": -0.6084710359573364,
      "logps/chosen": -194.9524688720703,
      "logps/rejected": -232.9259490966797,
      "loss": 0.584,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.22763672471046448,
      "rewards/margins": 0.2767763137817383,
      "rewards/rejected": -0.5044130086898804,
      "step": 25
    },
    {
      "epoch": 0.04,
      "learning_rate": 4.997807075247146e-05,
      "logits/chosen": -0.6930967569351196,
      "logits/rejected": -0.624418318271637,
      "logps/chosen": -201.62631225585938,
      "logps/rejected": -231.8871307373047,
      "loss": 0.5054,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.42100492119789124,
      "rewards/margins": 0.5007229447364807,
      "rewards/rejected": -0.9217279553413391,
      "step": 30
    },
    {
      "epoch": 0.05,
      "learning_rate": 4.996954133655838e-05,
      "logits/chosen": -0.6575398445129395,
      "logits/rejected": -0.5734818577766418,
      "logps/chosen": -215.608154296875,
      "logps/rejected": -259.5364990234375,
      "loss": 0.491,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.129244089126587,
      "rewards/margins": 0.6850830316543579,
      "rewards/rejected": -1.8143272399902344,
      "step": 35
    },
    {
      "epoch": 0.06,
      "learning_rate": 4.9959614877867264e-05,
      "logits/chosen": -0.5865448713302612,
      "logits/rejected": -0.5297074317932129,
      "logps/chosen": -251.30545043945312,
      "logps/rejected": -282.8074035644531,
      "loss": 0.4294,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.6625725030899048,
      "rewards/margins": 0.9844868779182434,
      "rewards/rejected": -2.647059440612793,
      "step": 40
    },
    {
      "epoch": 0.06,
      "learning_rate": 4.994829193178228e-05,
      "logits/chosen": -0.6305592656135559,
      "logits/rejected": -0.5662962794303894,
      "logps/chosen": -238.34683227539062,
      "logps/rejected": -288.0771789550781,
      "loss": 0.4195,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0896081924438477,
      "rewards/margins": 1.2067267894744873,
      "rewards/rejected": -3.296334743499756,
      "step": 45
    },
    {
      "epoch": 0.07,
      "learning_rate": 4.9935573131820854e-05,
      "logits/chosen": -0.5488190054893494,
      "logits/rejected": -0.4458453059196472,
      "logps/chosen": -237.5546417236328,
      "logps/rejected": -296.1310119628906,
      "loss": 0.2968,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.8281638622283936,
      "rewards/margins": 1.8660491704940796,
      "rewards/rejected": -4.694213390350342,
      "step": 50
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.99214591895983e-05,
      "logits/chosen": -0.496940940618515,
      "logits/rejected": -0.39123407006263733,
      "logps/chosen": -235.4664764404297,
      "logps/rejected": -301.9604797363281,
      "loss": 0.3675,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -4.971009731292725,
      "rewards/margins": 2.1594393253326416,
      "rewards/rejected": -7.130448818206787,
      "step": 55
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.990595089478795e-05,
      "logits/chosen": -0.44920119643211365,
      "logits/rejected": -0.3571397066116333,
      "logps/chosen": -270.9496154785156,
      "logps/rejected": -346.5156555175781,
      "loss": 0.3613,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.407833576202393,
      "rewards/margins": 2.685551166534424,
      "rewards/rejected": -8.093385696411133,
      "step": 60
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.9889049115077005e-05,
      "logits/chosen": -0.5083180665969849,
      "logits/rejected": -0.4510710835456848,
      "logps/chosen": -266.8662109375,
      "logps/rejected": -305.4253845214844,
      "loss": 0.4624,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.9966888427734375,
      "rewards/margins": 2.306278705596924,
      "rewards/rejected": -8.302966117858887,
      "step": 65
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.987075479611796e-05,
      "logits/chosen": -0.45440906286239624,
      "logits/rejected": -0.38472697138786316,
      "logps/chosen": -277.6520690917969,
      "logps/rejected": -340.0121765136719,
      "loss": 0.4809,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -6.467832088470459,
      "rewards/margins": 2.306380271911621,
      "rewards/rejected": -8.774212837219238,
      "step": 70
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9851068961475725e-05,
      "logits/chosen": -0.44587111473083496,
      "logits/rejected": -0.3460499048233032,
      "logps/chosen": -248.66030883789062,
      "logps/rejected": -324.4167785644531,
      "loss": 0.3326,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.31089973449707,
      "rewards/margins": 2.787933826446533,
      "rewards/rejected": -8.098833084106445,
      "step": 75
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.982999271257033e-05,
      "logits/chosen": -0.47116953134536743,
      "logits/rejected": -0.40097618103027344,
      "logps/chosen": -300.4339904785156,
      "logps/rejected": -352.66363525390625,
      "loss": 0.3232,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.735043525695801,
      "rewards/margins": 2.7127461433410645,
      "rewards/rejected": -8.447790145874023,
      "step": 80
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.980752722861531e-05,
      "logits/chosen": -0.5418936014175415,
      "logits/rejected": -0.48130369186401367,
      "logps/chosen": -273.4432067871094,
      "logps/rejected": -305.4323425292969,
      "loss": 0.4748,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -6.1163129806518555,
      "rewards/margins": 1.9569565057754517,
      "rewards/rejected": -8.07326889038086,
      "step": 85
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.978367376655177e-05,
      "logits/chosen": -0.6019693613052368,
      "logits/rejected": -0.501119077205658,
      "logps/chosen": -241.8748016357422,
      "logps/rejected": -303.2873840332031,
      "loss": 0.3463,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.80649471282959,
      "rewards/margins": 2.6070265769958496,
      "rewards/rejected": -7.413522243499756,
      "step": 90
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9758433660977964e-05,
      "logits/chosen": -0.5586158633232117,
      "logits/rejected": -0.46125540137290955,
      "logps/chosen": -253.9844512939453,
      "logps/rejected": -318.15960693359375,
      "loss": 0.2994,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -4.774440288543701,
      "rewards/margins": 2.6352882385253906,
      "rewards/rejected": -7.40972900390625,
      "step": 95
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9731808324074717e-05,
      "logits/chosen": -0.5857738256454468,
      "logits/rejected": -0.5078204274177551,
      "logps/chosen": -244.3525390625,
      "logps/rejected": -299.4289855957031,
      "loss": 0.4066,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.955862998962402,
      "rewards/margins": 2.2310631275177,
      "rewards/rejected": -7.186926364898682,
      "step": 100
    },
    {
      "epoch": 0.14,
      "eval_logits/chosen": -0.6736606359481812,
      "eval_logits/rejected": -0.5934695601463318,
      "eval_logps/chosen": -269.2135009765625,
      "eval_logps/rejected": -330.87744140625,
      "eval_loss": 0.36347857117652893,
      "eval_rewards/accuracies": 0.8051801919937134,
      "eval_rewards/chosen": -5.528914451599121,
      "eval_rewards/margins": 2.5316162109375,
      "eval_rewards/rejected": -8.060530662536621,
      "eval_runtime": 618.2599,
      "eval_samples_per_second": 0.954,
      "eval_steps_per_second": 0.12,
      "step": 100
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.970379924552635e-05,
      "logits/chosen": -0.6208111047744751,
      "logits/rejected": -0.5168194770812988,
      "logps/chosen": -256.57330322265625,
      "logps/rejected": -336.573486328125,
      "loss": 0.2796,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.576841354370117,
      "rewards/margins": 2.8258020877838135,
      "rewards/rejected": -8.402643203735352,
      "step": 105
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9674407992437394e-05,
      "logits/chosen": -0.5667734742164612,
      "logits/rejected": -0.5014970302581787,
      "logps/chosen": -279.5911560058594,
      "logps/rejected": -323.6059875488281,
      "loss": 0.3282,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.653090476989746,
      "rewards/margins": 2.4938788414001465,
      "rewards/rejected": -9.14696979522705,
      "step": 110
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.964363620924485e-05,
      "logits/chosen": -0.5388852953910828,
      "logits/rejected": -0.4672119617462158,
      "logps/chosen": -275.47357177734375,
      "logps/rejected": -326.81182861328125,
      "loss": 0.4124,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -6.149296760559082,
      "rewards/margins": 2.311483860015869,
      "rewards/rejected": -8.460780143737793,
      "step": 115
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.961148561762622e-05,
      "logits/chosen": -0.5033109784126282,
      "logits/rejected": -0.405276358127594,
      "logps/chosen": -256.9668884277344,
      "logps/rejected": -325.78839111328125,
      "loss": 0.3181,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.381417274475098,
      "rewards/margins": 2.8026371002197266,
      "rewards/rejected": -9.184053421020508,
      "step": 120
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9577958016403156e-05,
      "logits/chosen": -0.5019578337669373,
      "logits/rejected": -0.42507410049438477,
      "logps/chosen": -298.7206115722656,
      "logps/rejected": -352.7537841796875,
      "loss": 0.4869,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -6.763588905334473,
      "rewards/margins": 2.4632394313812256,
      "rewards/rejected": -9.226827621459961,
      "step": 125
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.954305528144085e-05,
      "logits/chosen": -0.5088043808937073,
      "logits/rejected": -0.40408769249916077,
      "logps/chosen": -260.7892761230469,
      "logps/rejected": -330.84747314453125,
      "loss": 0.2442,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.382019996643066,
      "rewards/margins": 3.018307685852051,
      "rewards/rejected": -9.400327682495117,
      "step": 130
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.9506779365543046e-05,
      "logits/chosen": -0.4910973608493805,
      "logits/rejected": -0.4001993238925934,
      "logps/chosen": -273.5351867675781,
      "logps/rejected": -341.2701110839844,
      "loss": 0.3465,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -6.231161117553711,
      "rewards/margins": 2.6937344074249268,
      "rewards/rejected": -8.924895286560059,
      "step": 135
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.946913229834279e-05,
      "logits/chosen": -0.5775797367095947,
      "logits/rejected": -0.48806411027908325,
      "logps/chosen": -259.29351806640625,
      "logps/rejected": -339.4101257324219,
      "loss": 0.3707,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -6.118729114532471,
      "rewards/margins": 3.231736421585083,
      "rewards/rejected": -9.350464820861816,
      "step": 140
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.943011618618887e-05,
      "logits/chosen": -0.4882396161556244,
      "logits/rejected": -0.4096860885620117,
      "logps/chosen": -258.9165954589844,
      "logps/rejected": -323.74822998046875,
      "loss": 0.3093,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.742823600769043,
      "rewards/margins": 3.2949295043945312,
      "rewards/rejected": -9.037752151489258,
      "step": 145
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.938973321202799e-05,
      "logits/chosen": -0.5276007056236267,
      "logits/rejected": -0.4654686450958252,
      "logps/chosen": -320.4021301269531,
      "logps/rejected": -362.60992431640625,
      "loss": 0.4327,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -7.360659122467041,
      "rewards/margins": 2.3584816455841064,
      "rewards/rejected": -9.719141006469727,
      "step": 150
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.93479856352826e-05,
      "logits/chosen": -0.6080215573310852,
      "logits/rejected": -0.561368465423584,
      "logps/chosen": -260.1334533691406,
      "logps/rejected": -304.76434326171875,
      "loss": 0.4775,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -5.641587257385254,
      "rewards/margins": 2.074352264404297,
      "rewards/rejected": -7.715939521789551,
      "step": 155
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.930487579172451e-05,
      "logits/chosen": -0.6534048318862915,
      "logits/rejected": -0.557198166847229,
      "logps/chosen": -238.5316925048828,
      "logps/rejected": -303.6593017578125,
      "loss": 0.3279,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.701598167419434,
      "rewards/margins": 2.6716597080230713,
      "rewards/rejected": -7.373257637023926,
      "step": 160
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.926040609334418e-05,
      "logits/chosen": -0.5949841141700745,
      "logits/rejected": -0.4924190938472748,
      "logps/chosen": -268.88604736328125,
      "logps/rejected": -336.0675048828125,
      "loss": 0.3907,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.271938800811768,
      "rewards/margins": 2.7763962745666504,
      "rewards/rejected": -8.048335075378418,
      "step": 165
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9214579028215776e-05,
      "logits/chosen": -0.5787319540977478,
      "logits/rejected": -0.5097864270210266,
      "logps/chosen": -270.76904296875,
      "logps/rejected": -330.5365905761719,
      "loss": 0.4062,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.298426628112793,
      "rewards/margins": 2.6809771060943604,
      "rewards/rejected": -7.979403495788574,
      "step": 170
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9167397160358e-05,
      "logits/chosen": -0.5979365110397339,
      "logits/rejected": -0.5267354846000671,
      "logps/chosen": -250.3006591796875,
      "logps/rejected": -310.52764892578125,
      "loss": 0.3127,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.677347660064697,
      "rewards/margins": 2.7255032062530518,
      "rewards/rejected": -8.402850151062012,
      "step": 175
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.911886312959055e-05,
      "logits/chosen": -0.600624144077301,
      "logits/rejected": -0.5323041677474976,
      "logps/chosen": -263.458984375,
      "logps/rejected": -324.83404541015625,
      "loss": 0.2828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.505527496337891,
      "rewards/margins": 2.577638626098633,
      "rewards/rejected": -8.083166122436523,
      "step": 180
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.906897965138653e-05,
      "logits/chosen": -0.6335456371307373,
      "logits/rejected": -0.5383679270744324,
      "logps/chosen": -271.63812255859375,
      "logps/rejected": -345.01702880859375,
      "loss": 0.3611,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -6.672010898590088,
      "rewards/margins": 3.4995288848876953,
      "rewards/rejected": -10.171539306640625,
      "step": 185
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.90177495167204e-05,
      "logits/chosen": -0.6199727058410645,
      "logits/rejected": -0.5486131906509399,
      "logps/chosen": -282.41583251953125,
      "logps/rejected": -347.9653625488281,
      "loss": 0.4037,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -6.799245357513428,
      "rewards/margins": 2.8858962059020996,
      "rewards/rejected": -9.685141563415527,
      "step": 190
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.896517559191194e-05,
      "logits/chosen": -0.683984100818634,
      "logits/rejected": -0.5775423049926758,
      "logps/chosen": -262.58538818359375,
      "logps/rejected": -335.623779296875,
      "loss": 0.4663,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.013031959533691,
      "rewards/margins": 3.129578113555908,
      "rewards/rejected": -10.142609596252441,
      "step": 195
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.891126081846579e-05,
      "logits/chosen": -0.6567696332931519,
      "logits/rejected": -0.5673569440841675,
      "logps/chosen": -256.83892822265625,
      "logps/rejected": -325.61065673828125,
      "loss": 0.3115,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.3370866775512695,
      "rewards/margins": 3.1089577674865723,
      "rewards/rejected": -9.446043968200684,
      "step": 200
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": -0.7238260507583618,
      "eval_logits/rejected": -0.6414477229118347,
      "eval_logps/chosen": -288.9779968261719,
      "eval_logps/rejected": -360.12066650390625,
      "eval_loss": 0.3467591404914856,
      "eval_rewards/accuracies": 0.832207202911377,
      "eval_rewards/chosen": -7.505363464355469,
      "eval_rewards/margins": 3.479483127593994,
      "eval_rewards/rejected": -10.984847068786621,
      "eval_runtime": 617.3232,
      "eval_samples_per_second": 0.956,
      "eval_steps_per_second": 0.12,
      "step": 200
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.8856008212906925e-05,
      "logits/chosen": -0.7143681049346924,
      "logits/rejected": -0.6183390021324158,
      "logps/chosen": -269.6623229980469,
      "logps/rejected": -350.52459716796875,
      "loss": 0.2657,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.174610137939453,
      "rewards/margins": 3.6835849285125732,
      "rewards/rejected": -10.858195304870605,
      "step": 205
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.879942086661184e-05,
      "logits/chosen": -0.6396993398666382,
      "logits/rejected": -0.5612179040908813,
      "logps/chosen": -320.8450622558594,
      "logps/rejected": -394.2232360839844,
      "loss": 0.2258,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -8.637298583984375,
      "rewards/margins": 3.9497833251953125,
      "rewards/rejected": -12.587082862854004,
      "step": 210
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.8741501945635656e-05,
      "logits/chosen": -0.6102300882339478,
      "logits/rejected": -0.5374587178230286,
      "logps/chosen": -261.81256103515625,
      "logps/rejected": -329.33453369140625,
      "loss": 0.3376,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.931007385253906,
      "rewards/margins": 3.3228728771209717,
      "rewards/rejected": -11.253880500793457,
      "step": 215
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.8682254690534876e-05,
      "logits/chosen": -0.589969277381897,
      "logits/rejected": -0.5134058594703674,
      "logps/chosen": -284.9112243652344,
      "logps/rejected": -351.90643310546875,
      "loss": 0.3639,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -7.687941551208496,
      "rewards/margins": 3.43585205078125,
      "rewards/rejected": -11.123793601989746,
      "step": 220
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.862168241618617e-05,
      "logits/chosen": -0.5980569124221802,
      "logits/rejected": -0.5199486017227173,
      "logps/chosen": -243.0383758544922,
      "logps/rejected": -318.08319091796875,
      "loss": 0.371,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.680181980133057,
      "rewards/margins": 3.751661777496338,
      "rewards/rejected": -10.431844711303711,
      "step": 225
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.8559788511600876e-05,
      "logits/chosen": -0.5468825101852417,
      "logits/rejected": -0.46182718873023987,
      "logps/chosen": -300.0223388671875,
      "logps/rejected": -365.42559814453125,
      "loss": 0.3014,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -8.608762741088867,
      "rewards/margins": 3.9094433784484863,
      "rewards/rejected": -12.518205642700195,
      "step": 230
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.849657643973535e-05,
      "logits/chosen": -0.5835626721382141,
      "logits/rejected": -0.46411094069480896,
      "logps/chosen": -289.1901550292969,
      "logps/rejected": -391.5763244628906,
      "loss": 0.2262,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.415358066558838,
      "rewards/margins": 4.35217809677124,
      "rewards/rejected": -11.767536163330078,
      "step": 235
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.843204973729729e-05,
      "logits/chosen": -0.5552169680595398,
      "logits/rejected": -0.4667941927909851,
      "logps/chosen": -262.4269714355469,
      "logps/rejected": -344.49462890625,
      "loss": 0.318,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -6.991745948791504,
      "rewards/margins": 3.809674024581909,
      "rewards/rejected": -10.801420211791992,
      "step": 240
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.8366212014547775e-05,
      "logits/chosen": -0.5869508981704712,
      "logits/rejected": -0.536375880241394,
      "logps/chosen": -263.1332092285156,
      "logps/rejected": -323.9794006347656,
      "loss": 0.3479,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -6.706090450286865,
      "rewards/margins": 3.0286171436309814,
      "rewards/rejected": -9.734708786010742,
      "step": 245
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.8299066955099335e-05,
      "logits/chosen": -0.6492255330085754,
      "logits/rejected": -0.572007954120636,
      "logps/chosen": -283.6471862792969,
      "logps/rejected": -338.55535888671875,
      "loss": 0.5536,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -6.6781325340271,
      "rewards/margins": 1.9866206645965576,
      "rewards/rejected": -8.664752960205078,
      "step": 250
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.823061831570981e-05,
      "logits/chosen": -0.5709847807884216,
      "logits/rejected": -0.49480828642845154,
      "logps/chosen": -280.56005859375,
      "logps/rejected": -343.18511962890625,
      "loss": 0.3881,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -6.904370307922363,
      "rewards/margins": 2.5633044242858887,
      "rewards/rejected": -9.46767520904541,
      "step": 255
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.816086992607219e-05,
      "logits/chosen": -0.5884816646575928,
      "logits/rejected": -0.48144420981407166,
      "logps/chosen": -259.3785095214844,
      "logps/rejected": -330.3200988769531,
      "loss": 0.3031,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.253561973571777,
      "rewards/margins": 2.664248466491699,
      "rewards/rejected": -8.917810440063477,
      "step": 260
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.8089825688600324e-05,
      "logits/chosen": -0.6093907356262207,
      "logits/rejected": -0.5078362226486206,
      "logps/chosen": -264.2926025390625,
      "logps/rejected": -339.7705993652344,
      "loss": 0.2332,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.6783013343811035,
      "rewards/margins": 3.519818067550659,
      "rewards/rejected": -10.198118209838867,
      "step": 265
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.8017489578210604e-05,
      "logits/chosen": -0.5917445421218872,
      "logits/rejected": -0.49844178557395935,
      "logps/chosen": -271.13677978515625,
      "logps/rejected": -333.9804382324219,
      "loss": 0.4124,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -7.525943756103516,
      "rewards/margins": 2.7923340797424316,
      "rewards/rejected": -10.318277359008789,
      "step": 270
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.794386564209953e-05,
      "logits/chosen": -0.6364978551864624,
      "logits/rejected": -0.5267010927200317,
      "logps/chosen": -282.91021728515625,
      "logps/rejected": -378.1858825683594,
      "loss": 0.2063,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.725545406341553,
      "rewards/margins": 4.508333206176758,
      "rewards/rejected": -11.233880043029785,
      "step": 275
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.786895799951732e-05,
      "logits/chosen": -0.5923280119895935,
      "logits/rejected": -0.512219250202179,
      "logps/chosen": -285.5169372558594,
      "logps/rejected": -352.58148193359375,
      "loss": 0.3159,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -8.174338340759277,
      "rewards/margins": 4.347126007080078,
      "rewards/rejected": -12.521463394165039,
      "step": 280
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.779277084153737e-05,
      "logits/chosen": -0.5970318913459778,
      "logits/rejected": -0.4951511025428772,
      "logps/chosen": -299.33294677734375,
      "logps/rejected": -385.574462890625,
      "loss": 0.3647,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -8.652000427246094,
      "rewards/margins": 3.701908588409424,
      "rewards/rejected": -12.353909492492676,
      "step": 285
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.7715308430821864e-05,
      "logits/chosen": -0.5532945394515991,
      "logits/rejected": -0.470702588558197,
      "logps/chosen": -289.81219482421875,
      "logps/rejected": -367.1038513183594,
      "loss": 0.3418,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -8.14123821258545,
      "rewards/margins": 4.235669136047363,
      "rewards/rejected": -12.376907348632812,
      "step": 290
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.765242323034498e-05,
      "logits/chosen": -0.5983850359916687,
      "logits/rejected": -0.5276827216148376,
      "logps/chosen": -291.8949890136719,
      "logps/rejected": -367.7808532714844,
      "loss": 0.2942,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.658424377441406,
      "rewards/margins": 4.356478691101074,
      "rewards/rejected": -11.01490306854248,
      "step": 295
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.757267633420931e-05,
      "logits/chosen": -0.5903723239898682,
      "logits/rejected": -0.470381498336792,
      "logps/chosen": -245.4473876953125,
      "logps/rejected": -322.921630859375,
      "loss": 0.2653,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.816718578338623,
      "rewards/margins": 3.5717501640319824,
      "rewards/rejected": -9.388467788696289,
      "step": 300
    },
    {
      "epoch": 0.43,
      "eval_logits/chosen": -0.6423404216766357,
      "eval_logits/rejected": -0.5549700856208801,
      "eval_logps/chosen": -277.4593811035156,
      "eval_logps/rejected": -346.16680908203125,
      "eval_loss": 0.3175624907016754,
      "eval_rewards/accuracies": 0.8372747302055359,
      "eval_rewards/chosen": -6.353503227233887,
      "eval_rewards/margins": 3.2359619140625,
      "eval_rewards/rejected": -9.589465141296387,
      "eval_runtime": 619.6776,
      "eval_samples_per_second": 0.952,
      "eval_steps_per_second": 0.119,
      "step": 300
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.7491666499598794e-05,
      "logits/chosen": -0.4752650856971741,
      "logits/rejected": -0.4291561245918274,
      "logps/chosen": -308.3399963378906,
      "logps/rejected": -351.7706604003906,
      "loss": 0.2566,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.376312255859375,
      "rewards/margins": 2.960242986679077,
      "rewards/rejected": -9.336555480957031,
      "step": 305
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.7409398259003744e-05,
      "logits/chosen": -0.5722283124923706,
      "logits/rejected": -0.43486547470092773,
      "logps/chosen": -271.27618408203125,
      "logps/rejected": -350.6097412109375,
      "loss": 0.3905,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.20401668548584,
      "rewards/margins": 3.1277387142181396,
      "rewards/rejected": -9.331754684448242,
      "step": 310
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.732587621532214e-05,
      "logits/chosen": -0.4831075668334961,
      "logits/rejected": -0.3779456317424774,
      "logps/chosen": -275.4000549316406,
      "logps/rejected": -348.45404052734375,
      "loss": 0.3821,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -7.6802659034729,
      "rewards/margins": 2.4858384132385254,
      "rewards/rejected": -10.166104316711426,
      "step": 315
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.724110504160208e-05,
      "logits/chosen": -0.42866721749305725,
      "logits/rejected": -0.39401277899742126,
      "logps/chosen": -322.8934020996094,
      "logps/rejected": -362.2698669433594,
      "loss": 0.3577,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -8.4872407913208,
      "rewards/margins": 2.9192988872528076,
      "rewards/rejected": -11.406539916992188,
      "step": 320
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.715508948078037e-05,
      "logits/chosen": -0.43140649795532227,
      "logits/rejected": -0.382727712392807,
      "logps/chosen": -309.9305114746094,
      "logps/rejected": -364.6683044433594,
      "loss": 0.4905,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -8.847053527832031,
      "rewards/margins": 2.7970926761627197,
      "rewards/rejected": -11.644145965576172,
      "step": 325
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.706783434541708e-05,
      "logits/chosen": -0.43298617005348206,
      "logits/rejected": -0.35435712337493896,
      "logps/chosen": -274.5734558105469,
      "logps/rejected": -323.9297790527344,
      "loss": 0.3598,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -7.113492012023926,
      "rewards/margins": 2.833127737045288,
      "rewards/rejected": -9.946619033813477,
      "step": 330
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.6979344517426345e-05,
      "logits/chosen": -0.44348543882369995,
      "logits/rejected": -0.3648318350315094,
      "logps/chosen": -311.8634338378906,
      "logps/rejected": -370.2818298339844,
      "loss": 0.3383,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -8.36948299407959,
      "rewards/margins": 2.6521434783935547,
      "rewards/rejected": -11.021625518798828,
      "step": 335
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.6889624947803195e-05,
      "logits/chosen": -0.4924314618110657,
      "logits/rejected": -0.3927859663963318,
      "logps/chosen": -286.1222229003906,
      "logps/rejected": -348.00445556640625,
      "loss": 0.2847,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.249931812286377,
      "rewards/margins": 3.2729618549346924,
      "rewards/rejected": -9.522893905639648,
      "step": 340
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.679868065634656e-05,
      "logits/chosen": -0.4742973744869232,
      "logits/rejected": -0.3609599769115448,
      "logps/chosen": -267.9313049316406,
      "logps/rejected": -338.3818359375,
      "loss": 0.2471,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.949034690856934,
      "rewards/margins": 3.703294277191162,
      "rewards/rejected": -10.652329444885254,
      "step": 345
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.6706516731378406e-05,
      "logits/chosen": -0.4882968068122864,
      "logits/rejected": -0.4227726459503174,
      "logps/chosen": -256.69964599609375,
      "logps/rejected": -336.7755432128906,
      "loss": 0.2902,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.316679954528809,
      "rewards/margins": 3.5853703022003174,
      "rewards/rejected": -9.902050971984863,
      "step": 350
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.661313832945904e-05,
      "logits/chosen": -0.45601844787597656,
      "logits/rejected": -0.35979634523391724,
      "logps/chosen": -297.2603759765625,
      "logps/rejected": -384.87481689453125,
      "loss": 0.2746,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -8.420430183410645,
      "rewards/margins": 4.463082313537598,
      "rewards/rejected": -12.883511543273926,
      "step": 355
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.65185506750986e-05,
      "logits/chosen": -0.39386382699012756,
      "logits/rejected": -0.28367191553115845,
      "logps/chosen": -309.7892150878906,
      "logps/rejected": -404.3554382324219,
      "loss": 0.3581,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -10.17359447479248,
      "rewards/margins": 4.7968430519104,
      "rewards/rejected": -14.970438003540039,
      "step": 360
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.642275906046475e-05,
      "logits/chosen": -0.42610305547714233,
      "logits/rejected": -0.32776302099227905,
      "logps/chosen": -308.56304931640625,
      "logps/rejected": -394.921142578125,
      "loss": 0.3434,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -9.843785285949707,
      "rewards/margins": 4.186667442321777,
      "rewards/rejected": -14.030451774597168,
      "step": 365
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.632576884508662e-05,
      "logits/chosen": -0.38670024275779724,
      "logits/rejected": -0.29175546765327454,
      "logps/chosen": -317.8299560546875,
      "logps/rejected": -382.1197814941406,
      "loss": 0.3862,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -10.289070129394531,
      "rewards/margins": 4.806598663330078,
      "rewards/rejected": -15.095669746398926,
      "step": 370
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.622758545555485e-05,
      "logits/chosen": -0.38157421350479126,
      "logits/rejected": -0.26931703090667725,
      "logps/chosen": -292.02069091796875,
      "logps/rejected": -388.94720458984375,
      "loss": 0.3453,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -7.471795558929443,
      "rewards/margins": 4.848240852355957,
      "rewards/rejected": -12.320035934448242,
      "step": 375
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.612821438521805e-05,
      "logits/chosen": -0.3492121398448944,
      "logits/rejected": -0.2813674807548523,
      "logps/chosen": -305.62554931640625,
      "logps/rejected": -356.89898681640625,
      "loss": 0.2206,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -8.157720565795898,
      "rewards/margins": 3.738807201385498,
      "rewards/rejected": -11.896527290344238,
      "step": 380
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.602766119387544e-05,
      "logits/chosen": -0.3653913736343384,
      "logits/rejected": -0.2597258687019348,
      "logps/chosen": -311.99371337890625,
      "logps/rejected": -382.8812255859375,
      "loss": 0.3092,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -8.370109558105469,
      "rewards/margins": 3.6346893310546875,
      "rewards/rejected": -12.004796981811523,
      "step": 385
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.592593150746576e-05,
      "logits/chosen": -0.3710058033466339,
      "logits/rejected": -0.2727561891078949,
      "logps/chosen": -288.8029479980469,
      "logps/rejected": -361.223388671875,
      "loss": 0.3955,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -7.695769309997559,
      "rewards/margins": 3.3667006492614746,
      "rewards/rejected": -11.062471389770508,
      "step": 390
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.5823031017752485e-05,
      "logits/chosen": -0.3490789532661438,
      "logits/rejected": -0.2746933102607727,
      "logps/chosen": -281.90704345703125,
      "logps/rejected": -338.3580322265625,
      "loss": 0.328,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -7.884057521820068,
      "rewards/margins": 2.856874942779541,
      "rewards/rejected": -10.740933418273926,
      "step": 395
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.571896548200542e-05,
      "logits/chosen": -0.4303611218929291,
      "logits/rejected": -0.3262333273887634,
      "logps/chosen": -261.8278503417969,
      "logps/rejected": -337.40521240234375,
      "loss": 0.2372,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.010989189147949,
      "rewards/margins": 3.444431781768799,
      "rewards/rejected": -9.45542049407959,
      "step": 400
    },
    {
      "epoch": 0.57,
      "eval_logits/chosen": -0.4868081510066986,
      "eval_logits/rejected": -0.3944084346294403,
      "eval_logps/chosen": -282.0663757324219,
      "eval_logps/rejected": -350.36456298828125,
      "eval_loss": 0.32044681906700134,
      "eval_rewards/accuracies": 0.8355855941772461,
      "eval_rewards/chosen": -6.814201831817627,
      "eval_rewards/margins": 3.195034980773926,
      "eval_rewards/rejected": -10.009236335754395,
      "eval_runtime": 635.268,
      "eval_samples_per_second": 0.929,
      "eval_steps_per_second": 0.116,
      "step": 400
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.5613740722678525e-05,
      "logits/chosen": -0.39522626996040344,
      "logits/rejected": -0.3030610680580139,
      "logps/chosen": -260.17266845703125,
      "logps/rejected": -320.7784118652344,
      "loss": 0.388,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -7.52327823638916,
      "rewards/margins": 3.621577739715576,
      "rewards/rejected": -11.144853591918945,
      "step": 405
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.550736262708418e-05,
      "logits/chosen": -0.4773409962654114,
      "logits/rejected": -0.3466936945915222,
      "logps/chosen": -277.6498107910156,
      "logps/rejected": -373.0572509765625,
      "loss": 0.2318,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.803755760192871,
      "rewards/margins": 3.537554979324341,
      "rewards/rejected": -10.34131145477295,
      "step": 410
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.5399837147063825e-05,
      "logits/chosen": -0.47985219955444336,
      "logits/rejected": -0.38592132925987244,
      "logps/chosen": -246.52841186523438,
      "logps/rejected": -332.86248779296875,
      "loss": 0.3023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.49072265625,
      "rewards/margins": 3.5066089630126953,
      "rewards/rejected": -9.997331619262695,
      "step": 415
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.529117029865488e-05,
      "logits/chosen": -0.4670870900154114,
      "logits/rejected": -0.36911553144454956,
      "logps/chosen": -260.58758544921875,
      "logps/rejected": -342.8556823730469,
      "loss": 0.3234,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.461057186126709,
      "rewards/margins": 4.485016822814941,
      "rewards/rejected": -11.946073532104492,
      "step": 420
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.518136816175419e-05,
      "logits/chosen": -0.5157058835029602,
      "logits/rejected": -0.4533557891845703,
      "logps/chosen": -296.40618896484375,
      "logps/rejected": -343.72515869140625,
      "loss": 0.4552,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -8.75401496887207,
      "rewards/margins": 2.8250133991241455,
      "rewards/rejected": -11.57902717590332,
      "step": 425
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.5070436879777865e-05,
      "logits/chosen": -0.5464299917221069,
      "logits/rejected": -0.4678220748901367,
      "logps/chosen": -253.18478393554688,
      "logps/rejected": -321.50714111328125,
      "loss": 0.3686,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -7.0356292724609375,
      "rewards/margins": 3.3276419639587402,
      "rewards/rejected": -10.36327075958252,
      "step": 430
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.495838265931754e-05,
      "logits/chosen": -0.5636313557624817,
      "logits/rejected": -0.4402475357055664,
      "logps/chosen": -268.5367126464844,
      "logps/rejected": -361.71697998046875,
      "loss": 0.2249,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.160154819488525,
      "rewards/margins": 3.917328357696533,
      "rewards/rejected": -11.077482223510742,
      "step": 435
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.4845211769793116e-05,
      "logits/chosen": -0.5847368240356445,
      "logits/rejected": -0.4880955219268799,
      "logps/chosen": -264.53619384765625,
      "logps/rejected": -339.5098571777344,
      "loss": 0.2691,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.8903937339782715,
      "rewards/margins": 3.62628173828125,
      "rewards/rejected": -11.51667594909668,
      "step": 440
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.473093054310199e-05,
      "logits/chosen": -0.5972886085510254,
      "logits/rejected": -0.5250357389450073,
      "logps/chosen": -300.294189453125,
      "logps/rejected": -370.1164855957031,
      "loss": 0.3729,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.556139945983887,
      "rewards/margins": 4.214568614959717,
      "rewards/rejected": -11.770709037780762,
      "step": 445
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.46155453732648e-05,
      "logits/chosen": -0.5448333024978638,
      "logits/rejected": -0.4635641574859619,
      "logps/chosen": -265.34027099609375,
      "logps/rejected": -342.3074035644531,
      "loss": 0.3015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -7.357121467590332,
      "rewards/margins": 4.111447334289551,
      "rewards/rejected": -11.468568801879883,
      "step": 450
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.449906271606766e-05,
      "logits/chosen": -0.5757399797439575,
      "logits/rejected": -0.5023797750473022,
      "logps/chosen": -283.24224853515625,
      "logps/rejected": -336.78485107421875,
      "loss": 0.309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -8.1708345413208,
      "rewards/margins": 3.2705204486846924,
      "rewards/rejected": -11.441353797912598,
      "step": 455
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.438148908870095e-05,
      "logits/chosen": -0.560847282409668,
      "logits/rejected": -0.4687212407588959,
      "logps/chosen": -262.0433349609375,
      "logps/rejected": -343.6734313964844,
      "loss": 0.2833,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -9.314498901367188,
      "rewards/margins": 3.861955165863037,
      "rewards/rejected": -13.176454544067383,
      "step": 460
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.426283106939474e-05,
      "logits/chosen": -0.5405054092407227,
      "logits/rejected": -0.4934763014316559,
      "logps/chosen": -344.86859130859375,
      "logps/rejected": -396.4712219238281,
      "loss": 0.3875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -11.307413101196289,
      "rewards/margins": 4.146143436431885,
      "rewards/rejected": -15.453557014465332,
      "step": 465
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.414309529705064e-05,
      "logits/chosen": -0.5560386776924133,
      "logits/rejected": -0.47137537598609924,
      "logps/chosen": -328.85467529296875,
      "logps/rejected": -410.0113830566406,
      "loss": 0.4106,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -10.061564445495605,
      "rewards/margins": 4.763692855834961,
      "rewards/rejected": -14.825258255004883,
      "step": 470
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.402228847087047e-05,
      "logits/chosen": -0.6119273900985718,
      "logits/rejected": -0.5192709565162659,
      "logps/chosen": -285.03509521484375,
      "logps/rejected": -359.7369689941406,
      "loss": 0.3005,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -9.684968948364258,
      "rewards/margins": 4.337248802185059,
      "rewards/rejected": -14.022216796875,
      "step": 475
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.390041734998132e-05,
      "logits/chosen": -0.6753469109535217,
      "logits/rejected": -0.5812332034111023,
      "logps/chosen": -275.0848083496094,
      "logps/rejected": -365.5320739746094,
      "loss": 0.2345,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.314253807067871,
      "rewards/margins": 4.565116882324219,
      "rewards/rejected": -11.879369735717773,
      "step": 480
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.3777488753057494e-05,
      "logits/chosen": -0.6396665573120117,
      "logits/rejected": -0.5838817358016968,
      "logps/chosen": -311.5262145996094,
      "logps/rejected": -385.2457580566406,
      "loss": 0.2286,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -8.526026725769043,
      "rewards/margins": 3.986698627471924,
      "rewards/rejected": -12.512724876403809,
      "step": 485
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.365350955793892e-05,
      "logits/chosen": -0.6760267019271851,
      "logits/rejected": -0.620951771736145,
      "logps/chosen": -313.78411865234375,
      "logps/rejected": -380.7740478515625,
      "loss": 0.3314,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -9.628170013427734,
      "rewards/margins": 3.3307273387908936,
      "rewards/rejected": -12.958897590637207,
      "step": 490
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.3528486701246376e-05,
      "logits/chosen": -0.6738308072090149,
      "logits/rejected": -0.5901960730552673,
      "logps/chosen": -285.67437744140625,
      "logps/rejected": -362.6622009277344,
      "loss": 0.3119,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -9.267045974731445,
      "rewards/margins": 3.928302049636841,
      "rewards/rejected": -13.195347785949707,
      "step": 495
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.3402427177993366e-05,
      "logits/chosen": -0.6766126155853271,
      "logits/rejected": -0.6122361421585083,
      "logps/chosen": -292.34326171875,
      "logps/rejected": -355.0762634277344,
      "loss": 0.4221,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -8.092927932739258,
      "rewards/margins": 4.039681911468506,
      "rewards/rejected": -12.132608413696289,
      "step": 500
    },
    {
      "epoch": 0.71,
      "eval_logits/chosen": -0.7289233803749084,
      "eval_logits/rejected": -0.6465120911598206,
      "eval_logps/chosen": -292.57330322265625,
      "eval_logps/rejected": -368.62078857421875,
      "eval_loss": 0.31609994173049927,
      "eval_rewards/accuracies": 0.8474099040031433,
      "eval_rewards/chosen": -7.864894390106201,
      "eval_rewards/margins": 3.9699699878692627,
      "eval_rewards/rejected": -11.83486557006836,
      "eval_runtime": 640.9294,
      "eval_samples_per_second": 0.921,
      "eval_steps_per_second": 0.115,
      "step": 500
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.327533804119476e-05,
      "logits/chosen": -0.6126315593719482,
      "logits/rejected": -0.5508357286453247,
      "logps/chosen": -282.2762756347656,
      "logps/rejected": -364.5578308105469,
      "loss": 0.316,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.242485046386719,
      "rewards/margins": 3.816096782684326,
      "rewards/rejected": -11.05858039855957,
      "step": 505
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.314722640147222e-05,
      "logits/chosen": -0.6599456071853638,
      "logits/rejected": -0.5939927697181702,
      "logps/chosen": -323.53668212890625,
      "logps/rejected": -396.10833740234375,
      "loss": 0.2941,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -8.693486213684082,
      "rewards/margins": 4.0846662521362305,
      "rewards/rejected": -12.778152465820312,
      "step": 510
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.301809942665625e-05,
      "logits/chosen": -0.6404463052749634,
      "logits/rejected": -0.5707911849021912,
      "logps/chosen": -307.43768310546875,
      "logps/rejected": -379.2852478027344,
      "loss": 0.4079,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -9.3197660446167,
      "rewards/margins": 3.961516857147217,
      "rewards/rejected": -13.281283378601074,
      "step": 515
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.28879643413853e-05,
      "logits/chosen": -0.65186607837677,
      "logits/rejected": -0.565552830696106,
      "logps/chosen": -310.2057189941406,
      "logps/rejected": -402.9633483886719,
      "loss": 0.2945,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -10.662870407104492,
      "rewards/margins": 4.87128210067749,
      "rewards/rejected": -15.534152030944824,
      "step": 520
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.2756828426701426e-05,
      "logits/chosen": -0.648743212223053,
      "logits/rejected": -0.5434762239456177,
      "logps/chosen": -288.9256591796875,
      "logps/rejected": -386.8144836425781,
      "loss": 0.2866,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -10.066654205322266,
      "rewards/margins": 5.4559478759765625,
      "rewards/rejected": -15.522601127624512,
      "step": 525
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.2624699019643e-05,
      "logits/chosen": -0.6845074892044067,
      "logits/rejected": -0.6216086149215698,
      "logps/chosen": -298.6322937011719,
      "logps/rejected": -368.7320861816406,
      "loss": 0.3515,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -8.717851638793945,
      "rewards/margins": 4.482119560241699,
      "rewards/rejected": -13.199971199035645,
      "step": 530
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.249158351283414e-05,
      "logits/chosen": -0.6760630011558533,
      "logits/rejected": -0.5973928570747375,
      "logps/chosen": -334.3371887207031,
      "logps/rejected": -412.73712158203125,
      "loss": 0.2374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -11.443408012390137,
      "rewards/margins": 5.742681503295898,
      "rewards/rejected": -17.18608856201172,
      "step": 535
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.235748935407115e-05,
      "logits/chosen": -0.6781556010246277,
      "logits/rejected": -0.6254957914352417,
      "logps/chosen": -314.7560729980469,
      "logps/rejected": -381.03436279296875,
      "loss": 0.571,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -10.918689727783203,
      "rewards/margins": 4.366137981414795,
      "rewards/rejected": -15.284828186035156,
      "step": 540
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.2222424045905774e-05,
      "logits/chosen": -0.703301727771759,
      "logits/rejected": -0.6157757639884949,
      "logps/chosen": -298.96417236328125,
      "logps/rejected": -374.37408447265625,
      "loss": 0.1956,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -10.03012752532959,
      "rewards/margins": 4.996109962463379,
      "rewards/rejected": -15.026239395141602,
      "step": 545
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.2086395145225456e-05,
      "logits/chosen": -0.6199958920478821,
      "logits/rejected": -0.5463480353355408,
      "logps/chosen": -297.76715087890625,
      "logps/rejected": -383.3893737792969,
      "loss": 0.3658,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -9.150928497314453,
      "rewards/margins": 4.323057651519775,
      "rewards/rejected": -13.473986625671387,
      "step": 550
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.1949410262830525e-05,
      "logits/chosen": -0.6586548686027527,
      "logits/rejected": -0.598033607006073,
      "logps/chosen": -298.9888610839844,
      "logps/rejected": -358.97113037109375,
      "loss": 0.3435,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -7.790317535400391,
      "rewards/margins": 3.196791172027588,
      "rewards/rejected": -10.987109184265137,
      "step": 555
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.1811477063008357e-05,
      "logits/chosen": -0.647259533405304,
      "logits/rejected": -0.5749470591545105,
      "logps/chosen": -283.01129150390625,
      "logps/rejected": -346.770263671875,
      "loss": 0.1992,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -8.713510513305664,
      "rewards/margins": 3.8825290203094482,
      "rewards/rejected": -12.596040725708008,
      "step": 560
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.167260326310459e-05,
      "logits/chosen": -0.655887246131897,
      "logits/rejected": -0.5856226682662964,
      "logps/chosen": -353.996337890625,
      "logps/rejected": -423.36199951171875,
      "loss": 0.3649,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -7.39592981338501,
      "rewards/margins": 4.420022010803223,
      "rewards/rejected": -11.81595230102539,
      "step": 565
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.1532796633091296e-05,
      "logits/chosen": -0.6630634069442749,
      "logits/rejected": -0.5999043583869934,
      "logps/chosen": -311.5198669433594,
      "logps/rejected": -382.8631896972656,
      "loss": 0.2865,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.376175880432129,
      "rewards/margins": 4.25052547454834,
      "rewards/rejected": -11.626702308654785,
      "step": 570
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.139206499513231e-05,
      "logits/chosen": -0.6371768712997437,
      "logits/rejected": -0.552124559879303,
      "logps/chosen": -315.57696533203125,
      "logps/rejected": -386.8127746582031,
      "loss": 0.3623,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -7.146079063415527,
      "rewards/margins": 3.366482973098755,
      "rewards/rejected": -10.51256275177002,
      "step": 575
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1250416223145515e-05,
      "logits/chosen": -0.6766015887260437,
      "logits/rejected": -0.5857943296432495,
      "logps/chosen": -259.1465759277344,
      "logps/rejected": -349.38995361328125,
      "loss": 0.3386,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -6.320498466491699,
      "rewards/margins": 3.3259353637695312,
      "rewards/rejected": -9.64643383026123,
      "step": 580
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.110785824236236e-05,
      "logits/chosen": -0.6103043556213379,
      "logits/rejected": -0.5319772958755493,
      "logps/chosen": -256.13446044921875,
      "logps/rejected": -331.8528137207031,
      "loss": 0.1824,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.32611608505249,
      "rewards/margins": 3.831328868865967,
      "rewards/rejected": -10.157445907592773,
      "step": 585
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.0964399028884394e-05,
      "logits/chosen": -0.6808601021766663,
      "logits/rejected": -0.5574377179145813,
      "logps/chosen": -255.42105102539062,
      "logps/rejected": -355.5065002441406,
      "loss": 0.2103,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.239567756652832,
      "rewards/margins": 4.258026599884033,
      "rewards/rejected": -10.497593879699707,
      "step": 590
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.082004660923703e-05,
      "logits/chosen": -0.6791261434555054,
      "logits/rejected": -0.5921913385391235,
      "logps/chosen": -267.8202819824219,
      "logps/rejected": -350.561767578125,
      "loss": 0.2129,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.478656768798828,
      "rewards/margins": 4.9322614669799805,
      "rewards/rejected": -12.410919189453125,
      "step": 595
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.067480905992045e-05,
      "logits/chosen": -0.7032105326652527,
      "logits/rejected": -0.600141167640686,
      "logps/chosen": -297.6030578613281,
      "logps/rejected": -384.7508850097656,
      "loss": 0.2602,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -9.610403060913086,
      "rewards/margins": 5.1422200202941895,
      "rewards/rejected": -14.7526216506958,
      "step": 600
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": -0.7646234631538391,
      "eval_logits/rejected": -0.6766347289085388,
      "eval_logps/chosen": -310.1993103027344,
      "eval_logps/rejected": -401.2223205566406,
      "eval_loss": 0.3386208117008209,
      "eval_rewards/accuracies": 0.8237612247467041,
      "eval_rewards/chosen": -9.627497673034668,
      "eval_rewards/margins": 5.467515468597412,
      "eval_rewards/rejected": -15.095013618469238,
      "eval_runtime": 635.6148,
      "eval_samples_per_second": 0.928,
      "eval_steps_per_second": 0.116,
      "step": 600
    }
  ],
  "max_steps": 2100,
  "num_train_epochs": 3,
  "total_flos": 6.092901381557453e+17,
  "trial_name": null,
  "trial_params": null
}