{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 7500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0004,
      "grad_norm": 55.133142230485284,
      "learning_rate": 6.666666666666666e-10,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.2421875,
      "logps/chosen": -129.0,
      "logps/rejected": -138.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.004,
      "grad_norm": 46.561242375212906,
      "learning_rate": 6.666666666666667e-09,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.4375,
      "logps/chosen": -157.0,
      "logps/rejected": -143.0,
      "loss": 0.6981,
      "rewards/accuracies": 0.1111111119389534,
      "rewards/chosen": -0.00555419921875,
      "rewards/margins": -0.02783203125,
      "rewards/rejected": 0.022216796875,
      "step": 10
    },
    {
      "epoch": 0.008,
      "grad_norm": 59.82198949811191,
      "learning_rate": 1.3333333333333334e-08,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.4453125,
      "logps/chosen": -151.0,
      "logps/rejected": -148.0,
      "loss": 0.6959,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": 0.0050048828125,
      "rewards/margins": -0.01251220703125,
      "rewards/rejected": 0.017578125,
      "step": 20
    },
    {
      "epoch": 0.012,
      "grad_norm": 51.97604614678016,
      "learning_rate": 2e-08,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.4453125,
      "logps/chosen": -133.0,
      "logps/rejected": -148.0,
      "loss": 0.6967,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01251220703125,
      "rewards/margins": -0.0125732421875,
      "rewards/rejected": 0.0,
      "step": 30
    },
    {
      "epoch": 0.016,
      "grad_norm": 58.35127549692606,
      "learning_rate": 2.6666666666666667e-08,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -145.0,
      "logps/rejected": -146.0,
      "loss": 0.6918,
      "rewards/accuracies": 0.10000000149011612,
      "rewards/chosen": -0.0050048828125,
      "rewards/margins": -0.00750732421875,
      "rewards/rejected": 0.0025177001953125,
      "step": 40
    },
    {
      "epoch": 0.02,
      "grad_norm": 55.332999749816075,
      "learning_rate": 3.3333333333333334e-08,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -127.0,
      "logps/rejected": -143.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.01129150390625,
      "rewards/margins": 0.01123046875,
      "rewards/rejected": -0.0224609375,
      "step": 50
    },
    {
      "epoch": 0.024,
      "grad_norm": 46.51922505461315,
      "learning_rate": 4e-08,
      "logits/chosen": -1.5,
      "logits/rejected": -1.4609375,
      "logps/chosen": -141.0,
      "logps/rejected": -149.0,
      "loss": 0.6967,
      "rewards/accuracies": 0.10000000149011612,
      "rewards/chosen": -0.021240234375,
      "rewards/margins": -0.0262451171875,
      "rewards/rejected": 0.0050048828125,
      "step": 60
    },
    {
      "epoch": 0.028,
      "grad_norm": 56.14098776919942,
      "learning_rate": 4.666666666666667e-08,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.40625,
      "logps/chosen": -137.0,
      "logps/rejected": -141.0,
      "loss": 0.6895,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.0238037109375,
      "rewards/margins": -0.0162353515625,
      "rewards/rejected": -0.00750732421875,
      "step": 70
    },
    {
      "epoch": 0.032,
      "grad_norm": 54.16190763797781,
      "learning_rate": 5.3333333333333334e-08,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.3984375,
      "logps/chosen": -128.0,
      "logps/rejected": -151.0,
      "loss": 0.6742,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.030029296875,
      "rewards/rejected": -0.030029296875,
      "step": 80
    },
    {
      "epoch": 0.036,
      "grad_norm": 54.75938888217226,
      "learning_rate": 6e-08,
      "logits/chosen": -1.5234375,
      "logits/rejected": -1.453125,
      "logps/chosen": -136.0,
      "logps/rejected": -154.0,
      "loss": 0.6654,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.003753662109375,
      "rewards/margins": 0.061279296875,
      "rewards/rejected": -0.06494140625,
      "step": 90
    },
    {
      "epoch": 0.04,
      "grad_norm": 55.767549697861355,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.5234375,
      "logps/chosen": -145.0,
      "logps/rejected": -150.0,
      "loss": 0.649,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 6.109476089477539e-06,
      "rewards/margins": 0.10009765625,
      "rewards/rejected": -0.10009765625,
      "step": 100
    },
    {
      "epoch": 0.044,
      "grad_norm": 51.22085363412276,
      "learning_rate": 7.333333333333333e-08,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.4765625,
      "logps/chosen": -188.0,
      "logps/rejected": -140.0,
      "loss": 0.6355,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.018798828125,
      "rewards/margins": 0.1513671875,
      "rewards/rejected": -0.1328125,
      "step": 110
    },
    {
      "epoch": 0.048,
      "grad_norm": 49.0358131256224,
      "learning_rate": 8e-08,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.2890625,
      "logps/chosen": -142.0,
      "logps/rejected": -142.0,
      "loss": 0.6252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.004974365234375,
      "rewards/margins": 0.1796875,
      "rewards/rejected": -0.1748046875,
      "step": 120
    },
    {
      "epoch": 0.052,
      "grad_norm": 46.5700635017445,
      "learning_rate": 8.666666666666666e-08,
      "logits/chosen": -1.5,
      "logits/rejected": -1.4375,
      "logps/chosen": -134.0,
      "logps/rejected": -142.0,
      "loss": 0.59,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.109476089477539e-06,
      "rewards/margins": 0.212890625,
      "rewards/rejected": -0.212890625,
      "step": 130
    },
    {
      "epoch": 0.056,
      "grad_norm": 45.23066482062717,
      "learning_rate": 9.333333333333334e-08,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.421875,
      "logps/chosen": -133.0,
      "logps/rejected": -151.0,
      "loss": 0.5691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.001251220703125,
      "rewards/margins": 0.27734375,
      "rewards/rejected": -0.275390625,
      "step": 140
    },
    {
      "epoch": 0.06,
      "grad_norm": 42.03733294933095,
      "learning_rate": 1e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.4375,
      "logps/chosen": -185.0,
      "logps/rejected": -147.0,
      "loss": 0.5368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0024871826171875,
      "rewards/margins": 0.349609375,
      "rewards/rejected": -0.34765625,
      "step": 150
    },
    {
      "epoch": 0.064,
      "grad_norm": 42.34285549047089,
      "learning_rate": 1.0666666666666667e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.34375,
      "logps/chosen": -151.0,
      "logps/rejected": -148.0,
      "loss": 0.5152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01007080078125,
      "rewards/margins": 0.384765625,
      "rewards/rejected": -0.39453125,
      "step": 160
    },
    {
      "epoch": 0.068,
      "grad_norm": 33.63660442706237,
      "learning_rate": 1.1333333333333332e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.2421875,
      "logps/chosen": -123.5,
      "logps/rejected": -146.0,
      "loss": 0.4763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02880859375,
      "rewards/margins": 0.47265625,
      "rewards/rejected": -0.50390625,
      "step": 170
    },
    {
      "epoch": 0.072,
      "grad_norm": 36.91547618772574,
      "learning_rate": 1.2e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.40625,
      "logps/chosen": -117.5,
      "logps/rejected": -150.0,
      "loss": 0.453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.054931640625,
      "rewards/margins": 0.5546875,
      "rewards/rejected": -0.609375,
      "step": 180
    },
    {
      "epoch": 0.076,
      "grad_norm": 32.807565830855744,
      "learning_rate": 1.2666666666666666e-07,
      "logits/chosen": -1.4765625,
      "logits/rejected": -1.375,
      "logps/chosen": -127.0,
      "logps/rejected": -154.0,
      "loss": 0.4137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03369140625,
      "rewards/margins": 0.62890625,
      "rewards/rejected": -0.66015625,
      "step": 190
    },
    {
      "epoch": 0.08,
      "grad_norm": 30.577412703087763,
      "learning_rate": 1.3333333333333334e-07,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -136.0,
      "logps/rejected": -162.0,
      "loss": 0.3564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.044921875,
      "rewards/margins": 0.83203125,
      "rewards/rejected": -0.875,
      "step": 200
    },
    {
      "epoch": 0.084,
      "grad_norm": 30.625844138300014,
      "learning_rate": 1.4e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.234375,
      "logps/chosen": -122.0,
      "logps/rejected": -153.0,
      "loss": 0.3471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03369140625,
      "rewards/margins": 0.88671875,
      "rewards/rejected": -0.91796875,
      "step": 210
    },
    {
      "epoch": 0.088,
      "grad_norm": 21.502557480079705,
      "learning_rate": 1.4666666666666666e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.3125,
      "logps/chosen": -159.0,
      "logps/rejected": -153.0,
      "loss": 0.2969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01251220703125,
      "rewards/margins": 1.078125,
      "rewards/rejected": -1.09375,
      "step": 220
    },
    {
      "epoch": 0.092,
      "grad_norm": 25.393015588011483,
      "learning_rate": 1.533333333333333e-07,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.3125,
      "logps/chosen": -165.0,
      "logps/rejected": -158.0,
      "loss": 0.255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01251220703125,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -1.34375,
      "step": 230
    },
    {
      "epoch": 0.096,
      "grad_norm": 17.772100247923397,
      "learning_rate": 1.6e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.359375,
      "logps/chosen": -140.0,
      "logps/rejected": -161.0,
      "loss": 0.2021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04638671875,
      "rewards/margins": 1.625,
      "rewards/rejected": -1.671875,
      "step": 240
    },
    {
      "epoch": 0.1,
      "grad_norm": 14.704628161007186,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.34375,
      "logps/chosen": -178.0,
      "logps/rejected": -160.0,
      "loss": 0.1728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0286865234375,
      "rewards/margins": 1.78125,
      "rewards/rejected": -1.8125,
      "step": 250
    },
    {
      "epoch": 0.104,
      "grad_norm": 16.336530967046354,
      "learning_rate": 1.7333333333333332e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.34375,
      "logps/chosen": -135.0,
      "logps/rejected": -164.0,
      "loss": 0.1389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1943359375,
      "rewards/margins": 2.125,
      "rewards/rejected": -2.3125,
      "step": 260
    },
    {
      "epoch": 0.108,
      "grad_norm": 17.2226442520799,
      "learning_rate": 1.8e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3359375,
      "logps/chosen": -121.5,
      "logps/rejected": -168.0,
      "loss": 0.1075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.291015625,
      "rewards/margins": 2.453125,
      "rewards/rejected": -2.734375,
      "step": 270
    },
    {
      "epoch": 0.112,
      "grad_norm": 5.785820771445246,
      "learning_rate": 1.8666666666666667e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.34375,
      "logps/chosen": -148.0,
      "logps/rejected": -173.0,
      "loss": 0.0789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25390625,
      "rewards/margins": 2.78125,
      "rewards/rejected": -3.03125,
      "step": 280
    },
    {
      "epoch": 0.116,
      "grad_norm": 3.0277795007065054,
      "learning_rate": 1.9333333333333332e-07,
      "logits/chosen": -1.5390625,
      "logits/rejected": -1.296875,
      "logps/chosen": -161.0,
      "logps/rejected": -195.0,
      "loss": 0.0355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5,
      "rewards/margins": 3.65625,
      "rewards/rejected": -4.15625,
      "step": 290
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.668641014068527,
      "learning_rate": 2e-07,
      "logits/chosen": -1.5,
      "logits/rejected": -1.3359375,
      "logps/chosen": -152.0,
      "logps/rejected": -188.0,
      "loss": 0.0291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.470703125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -4.9375,
      "step": 300
    },
    {
      "epoch": 0.124,
      "grad_norm": 2.0374131624809806,
      "learning_rate": 2.0666666666666666e-07,
      "logits/chosen": -1.4765625,
      "logits/rejected": -1.2734375,
      "logps/chosen": -173.0,
      "logps/rejected": -200.0,
      "loss": 0.0187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.341796875,
      "rewards/margins": 5.09375,
      "rewards/rejected": -5.4375,
      "step": 310
    },
    {
      "epoch": 0.128,
      "grad_norm": 0.293086841772368,
      "learning_rate": 2.1333333333333334e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -134.0,
      "logps/rejected": -202.0,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.52734375,
      "rewards/margins": 5.53125,
      "rewards/rejected": -6.0625,
      "step": 320
    },
    {
      "epoch": 0.132,
      "grad_norm": 0.3276363861927301,
      "learning_rate": 2.1999999999999998e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.3125,
      "logps/chosen": -174.0,
      "logps/rejected": -218.0,
      "loss": 0.0095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41796875,
      "rewards/margins": 6.375,
      "rewards/rejected": -6.8125,
      "step": 330
    },
    {
      "epoch": 0.136,
      "grad_norm": 2.464318417521699,
      "learning_rate": 2.2666666666666663e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -157.0,
      "logps/rejected": -203.0,
      "loss": 0.0144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7109375,
      "rewards/margins": 5.375,
      "rewards/rejected": -6.0625,
      "step": 340
    },
    {
      "epoch": 0.14,
      "grad_norm": 0.3760662875963674,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -144.0,
      "logps/rejected": -222.0,
      "loss": 0.0033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.796875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -7.375,
      "step": 350
    },
    {
      "epoch": 0.144,
      "grad_norm": 0.18358416685910736,
      "learning_rate": 2.4e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.234375,
      "logps/chosen": -133.0,
      "logps/rejected": -223.0,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.77734375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -8.0,
      "step": 360
    },
    {
      "epoch": 0.148,
      "grad_norm": 0.17524319278921732,
      "learning_rate": 2.4666666666666665e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -154.0,
      "logps/rejected": -232.0,
      "loss": 0.0034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.96484375,
      "rewards/margins": 7.25,
      "rewards/rejected": -8.1875,
      "step": 370
    },
    {
      "epoch": 0.152,
      "grad_norm": 0.11165712943537118,
      "learning_rate": 2.533333333333333e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.3515625,
      "logps/chosen": -183.0,
      "logps/rejected": -223.0,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.796875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -8.125,
      "step": 380
    },
    {
      "epoch": 0.156,
      "grad_norm": 0.19585698268831317,
      "learning_rate": 2.6e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.2890625,
      "logps/chosen": -134.0,
      "logps/rejected": -226.0,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -8.3125,
      "step": 390
    },
    {
      "epoch": 0.16,
      "grad_norm": 0.039601783559183756,
      "learning_rate": 2.6666666666666667e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4375,
      "logps/chosen": -142.0,
      "logps/rejected": -234.0,
      "loss": 0.0033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.96875,
      "rewards/margins": 7.78125,
      "rewards/rejected": -8.75,
      "step": 400
    },
    {
      "epoch": 0.164,
      "grad_norm": 0.0198244519839624,
      "learning_rate": 2.733333333333333e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.328125,
      "logps/chosen": -139.0,
      "logps/rejected": -233.0,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -8.4375,
      "step": 410
    },
    {
      "epoch": 0.168,
      "grad_norm": 0.028945529048788294,
      "learning_rate": 2.8e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.296875,
      "logps/chosen": -181.0,
      "logps/rejected": -238.0,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.015625,
      "rewards/margins": 8.1875,
      "rewards/rejected": -9.1875,
      "step": 420
    },
    {
      "epoch": 0.172,
      "grad_norm": 3.413786204395638,
      "learning_rate": 2.866666666666667e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.265625,
      "logps/chosen": -157.0,
      "logps/rejected": -242.0,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.98046875,
      "rewards/margins": 8.4375,
      "rewards/rejected": -9.375,
      "step": 430
    },
    {
      "epoch": 0.176,
      "grad_norm": 5.347686299164599,
      "learning_rate": 2.933333333333333e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.328125,
      "logps/chosen": -160.0,
      "logps/rejected": -234.0,
      "loss": 0.0282,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -9.0625,
      "step": 440
    },
    {
      "epoch": 0.18,
      "grad_norm": 0.02200966297762767,
      "learning_rate": 3e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.3671875,
      "logps/chosen": -181.0,
      "logps/rejected": -240.0,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.265625,
      "rewards/margins": 8.25,
      "rewards/rejected": -9.5,
      "step": 450
    },
    {
      "epoch": 0.184,
      "grad_norm": 0.10242874101385889,
      "learning_rate": 3.066666666666666e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.2265625,
      "logps/chosen": -143.0,
      "logps/rejected": -250.0,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0625,
      "rewards/margins": 9.0625,
      "rewards/rejected": -10.125,
      "step": 460
    },
    {
      "epoch": 0.188,
      "grad_norm": 0.017492380331089297,
      "learning_rate": 3.1333333333333333e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.296875,
      "logps/chosen": -140.0,
      "logps/rejected": -241.0,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.515625,
      "rewards/margins": 8.75,
      "rewards/rejected": -10.25,
      "step": 470
    },
    {
      "epoch": 0.192,
      "grad_norm": 0.018721777846966416,
      "learning_rate": 3.2e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.328125,
      "logps/chosen": -170.0,
      "logps/rejected": -260.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -10.875,
      "step": 480
    },
    {
      "epoch": 0.196,
      "grad_norm": 0.046539912588897746,
      "learning_rate": 3.2666666666666663e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.1875,
      "logps/chosen": -136.0,
      "logps/rejected": -248.0,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.875,
      "rewards/margins": 8.5625,
      "rewards/rejected": -10.4375,
      "step": 490
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.04408370053606096,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.3671875,
      "logps/chosen": -171.0,
      "logps/rejected": -250.0,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.46875,
      "rewards/margins": 9.125,
      "rewards/rejected": -10.5625,
      "step": 500
    },
    {
      "epoch": 0.204,
      "grad_norm": 1.0210919282120885,
      "learning_rate": 3.4000000000000003e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.3125,
      "logps/chosen": -170.0,
      "logps/rejected": -256.0,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -10.4375,
      "step": 510
    },
    {
      "epoch": 0.208,
      "grad_norm": 0.8571946233604452,
      "learning_rate": 3.4666666666666665e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.265625,
      "logps/chosen": -151.0,
      "logps/rejected": -260.0,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.59375,
      "rewards/margins": 9.6875,
      "rewards/rejected": -11.25,
      "step": 520
    },
    {
      "epoch": 0.212,
      "grad_norm": 0.019616994438370134,
      "learning_rate": 3.533333333333333e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.21875,
      "logps/chosen": -170.0,
      "logps/rejected": -258.0,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.421875,
      "rewards/margins": 9.75,
      "rewards/rejected": -11.1875,
      "step": 530
    },
    {
      "epoch": 0.216,
      "grad_norm": 0.21364606101941117,
      "learning_rate": 3.6e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.3203125,
      "logps/chosen": -133.0,
      "logps/rejected": -266.0,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -11.375,
      "step": 540
    },
    {
      "epoch": 0.22,
      "grad_norm": 1.3668188994451338,
      "learning_rate": 3.666666666666666e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -148.0,
      "logps/rejected": -266.0,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.359375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -11.6875,
      "step": 550
    },
    {
      "epoch": 0.224,
      "grad_norm": 0.005206201889455875,
      "learning_rate": 3.7333333333333334e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.265625,
      "logps/chosen": -125.0,
      "logps/rejected": -262.0,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -11.4375,
      "step": 560
    },
    {
      "epoch": 0.228,
      "grad_norm": 8.202061770588287,
      "learning_rate": 3.7999999999999996e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.3515625,
      "logps/chosen": -157.0,
      "logps/rejected": -262.0,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 10.4375,
      "rewards/rejected": -11.8125,
      "step": 570
    },
    {
      "epoch": 0.232,
      "grad_norm": 0.015307472966039197,
      "learning_rate": 3.8666666666666664e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.28125,
      "logps/chosen": -161.0,
      "logps/rejected": -264.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.265625,
      "rewards/margins": 10.5,
      "rewards/rejected": -11.75,
      "step": 580
    },
    {
      "epoch": 0.236,
      "grad_norm": 0.005487541785070303,
      "learning_rate": 3.933333333333333e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.3515625,
      "logps/chosen": -154.0,
      "logps/rejected": -270.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.65625,
      "rewards/margins": 10.75,
      "rewards/rejected": -12.4375,
      "step": 590
    },
    {
      "epoch": 0.24,
      "grad_norm": 0.9338808253816322,
      "learning_rate": 4e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.15625,
      "logps/chosen": -153.0,
      "logps/rejected": -264.0,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 10.5625,
      "rewards/rejected": -12.0,
      "step": 600
    },
    {
      "epoch": 0.244,
      "grad_norm": 0.01465602456728562,
      "learning_rate": 4.0666666666666666e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.1640625,
      "logps/chosen": -163.0,
      "logps/rejected": -286.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.546875,
      "rewards/margins": 12.125,
      "rewards/rejected": -13.6875,
      "step": 610
    },
    {
      "epoch": 0.248,
      "grad_norm": 0.004038228559617097,
      "learning_rate": 4.1333333333333333e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.3125,
      "logps/chosen": -172.0,
      "logps/rejected": -282.0,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 11.5625,
      "rewards/rejected": -13.5,
      "step": 620
    },
    {
      "epoch": 0.252,
      "grad_norm": 0.025518933690051763,
      "learning_rate": 4.1999999999999995e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.2265625,
      "logps/chosen": -178.0,
      "logps/rejected": -282.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -13.375,
      "step": 630
    },
    {
      "epoch": 0.256,
      "grad_norm": 0.00887110356163342,
      "learning_rate": 4.266666666666667e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -191.0,
      "logps/rejected": -280.0,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.03125,
      "rewards/margins": 11.375,
      "rewards/rejected": -13.4375,
      "step": 640
    },
    {
      "epoch": 0.26,
      "grad_norm": 0.00065018062207251,
      "learning_rate": 4.3333333333333335e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.3359375,
      "logps/chosen": -156.0,
      "logps/rejected": -286.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.859375,
      "rewards/margins": 12.125,
      "rewards/rejected": -13.9375,
      "step": 650
    },
    {
      "epoch": 0.264,
      "grad_norm": 0.0018112136976451473,
      "learning_rate": 4.3999999999999997e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.28125,
      "logps/chosen": -160.0,
      "logps/rejected": -292.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.046875,
      "rewards/margins": 11.75,
      "rewards/rejected": -13.75,
      "step": 660
    },
    {
      "epoch": 0.268,
      "grad_norm": 0.0037038010450957367,
      "learning_rate": 4.4666666666666664e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.2421875,
      "logps/chosen": -136.0,
      "logps/rejected": -276.0,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.546875,
      "rewards/margins": 11.625,
      "rewards/rejected": -14.1875,
      "step": 670
    },
    {
      "epoch": 0.272,
      "grad_norm": 0.003093992636072826,
      "learning_rate": 4.5333333333333326e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.2890625,
      "logps/chosen": -161.0,
      "logps/rejected": -292.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.203125,
      "rewards/margins": 12.25,
      "rewards/rejected": -14.4375,
      "step": 680
    },
    {
      "epoch": 0.276,
      "grad_norm": 0.002721132694652657,
      "learning_rate": 4.6e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -179.0,
      "logps/rejected": -298.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.453125,
      "rewards/margins": 12.4375,
      "rewards/rejected": -14.875,
      "step": 690
    },
    {
      "epoch": 0.28,
      "grad_norm": 0.004071505626352889,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.2421875,
      "logps/chosen": -158.0,
      "logps/rejected": -290.0,
      "loss": 0.0172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.90625,
      "rewards/margins": 11.875,
      "rewards/rejected": -14.8125,
      "step": 700
    },
    {
      "epoch": 0.284,
      "grad_norm": 0.008253743206850692,
      "learning_rate": 4.733333333333333e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.2890625,
      "logps/chosen": -194.0,
      "logps/rejected": -292.0,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.859375,
      "rewards/margins": 12.0,
      "rewards/rejected": -14.875,
      "step": 710
    },
    {
      "epoch": 0.288,
      "grad_norm": 0.0024921255376558235,
      "learning_rate": 4.8e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.1640625,
      "logps/chosen": -160.0,
      "logps/rejected": -298.0,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.546875,
      "rewards/margins": 12.3125,
      "rewards/rejected": -14.875,
      "step": 720
    },
    {
      "epoch": 0.292,
      "grad_norm": 0.00027377845155000493,
      "learning_rate": 4.866666666666666e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -198.0,
      "logps/rejected": -298.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5625,
      "rewards/margins": 12.5625,
      "rewards/rejected": -15.125,
      "step": 730
    },
    {
      "epoch": 0.296,
      "grad_norm": 0.010538882448411386,
      "learning_rate": 4.933333333333333e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.109375,
      "logps/chosen": -156.0,
      "logps/rejected": -298.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.890625,
      "rewards/margins": 12.125,
      "rewards/rejected": -15.0,
      "step": 740
    },
    {
      "epoch": 0.3,
      "grad_norm": 0.008986225850815419,
      "learning_rate": 5e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.1953125,
      "logps/chosen": -156.0,
      "logps/rejected": -300.0,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.75,
      "rewards/margins": 12.5625,
      "rewards/rejected": -15.3125,
      "step": 750
    },
    {
      "epoch": 0.304,
      "grad_norm": 0.008761345718463875,
      "learning_rate": 4.992592592592593e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.3515625,
      "logps/chosen": -167.0,
      "logps/rejected": -298.0,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.828125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -15.6875,
      "step": 760
    },
    {
      "epoch": 0.308,
      "grad_norm": 7.068282309538755e-05,
      "learning_rate": 4.985185185185185e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.140625,
      "logps/chosen": -191.0,
      "logps/rejected": -308.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.46875,
      "rewards/margins": 13.125,
      "rewards/rejected": -16.625,
      "step": 770
    },
    {
      "epoch": 0.312,
      "grad_norm": 0.0005898883097514736,
      "learning_rate": 4.977777777777777e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -1.2890625,
      "logps/chosen": -216.0,
      "logps/rejected": -304.0,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.953125,
      "rewards/margins": 12.75,
      "rewards/rejected": -15.6875,
      "step": 780
    },
    {
      "epoch": 0.316,
      "grad_norm": 0.013875528465489343,
      "learning_rate": 4.97037037037037e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.203125,
      "logps/chosen": -182.0,
      "logps/rejected": -302.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.21875,
      "rewards/margins": 12.4375,
      "rewards/rejected": -15.6875,
      "step": 790
    },
    {
      "epoch": 0.32,
      "grad_norm": 0.002411259567257776,
      "learning_rate": 4.962962962962963e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.1484375,
      "logps/chosen": -171.0,
      "logps/rejected": -312.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.390625,
      "rewards/margins": 13.1875,
      "rewards/rejected": -16.625,
      "step": 800
    },
    {
      "epoch": 0.324,
      "grad_norm": 0.0009656671077378089,
      "learning_rate": 4.955555555555556e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.2109375,
      "logps/chosen": -160.0,
      "logps/rejected": -314.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.34375,
      "rewards/margins": 13.3125,
      "rewards/rejected": -16.75,
      "step": 810
    },
    {
      "epoch": 0.328,
      "grad_norm": 0.0003325761352635712,
      "learning_rate": 4.948148148148148e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.234375,
      "logps/chosen": -194.0,
      "logps/rejected": -302.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.796875,
      "rewards/margins": 12.875,
      "rewards/rejected": -15.6875,
      "step": 820
    },
    {
      "epoch": 0.332,
      "grad_norm": 0.1987368800425745,
      "learning_rate": 4.94074074074074e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.328125,
      "logps/chosen": -191.0,
      "logps/rejected": -316.0,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.140625,
      "rewards/margins": 12.75,
      "rewards/rejected": -15.875,
      "step": 830
    },
    {
      "epoch": 0.336,
      "grad_norm": 0.09932120709043088,
      "learning_rate": 4.933333333333333e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.125,
      "logps/chosen": -200.0,
      "logps/rejected": -316.0,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.53125,
      "rewards/margins": 12.625,
      "rewards/rejected": -17.125,
      "step": 840
    },
    {
      "epoch": 0.34,
      "grad_norm": 0.00012339542154780997,
      "learning_rate": 4.925925925925926e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.2265625,
      "logps/chosen": -190.0,
      "logps/rejected": -308.0,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.1875,
      "rewards/margins": 11.625,
      "rewards/rejected": -15.875,
      "step": 850
    },
    {
      "epoch": 0.344,
      "grad_norm": 0.0017244209573168603,
      "learning_rate": 4.918518518518519e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.3359375,
      "logps/chosen": -172.0,
      "logps/rejected": -306.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.484375,
      "rewards/margins": 12.875,
      "rewards/rejected": -15.375,
      "step": 860
    },
    {
      "epoch": 0.348,
      "grad_norm": 0.028163037298035626,
      "learning_rate": 4.91111111111111e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.1953125,
      "logps/chosen": -154.0,
      "logps/rejected": -300.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.609375,
      "rewards/margins": 12.5625,
      "rewards/rejected": -15.1875,
      "step": 870
    },
    {
      "epoch": 0.352,
      "grad_norm": 0.004489526873074734,
      "learning_rate": 4.903703703703703e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.15625,
      "logps/chosen": -193.0,
      "logps/rejected": -308.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.515625,
      "rewards/margins": 13.1875,
      "rewards/rejected": -15.75,
      "step": 880
    },
    {
      "epoch": 0.356,
      "grad_norm": 0.0048165359644522345,
      "learning_rate": 4.896296296296296e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.25,
      "logps/chosen": -163.0,
      "logps/rejected": -310.0,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.609375,
      "rewards/margins": 12.875,
      "rewards/rejected": -16.5,
      "step": 890
    },
    {
      "epoch": 0.36,
      "grad_norm": 0.022355040398337484,
      "learning_rate": 4.888888888888889e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.25,
      "logps/chosen": -197.0,
      "logps/rejected": -314.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4375,
      "rewards/margins": 13.25,
      "rewards/rejected": -16.625,
      "step": 900
    },
    {
      "epoch": 0.364,
      "grad_norm": 0.00018203987263011996,
      "learning_rate": 4.881481481481482e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.171875,
      "logps/chosen": -187.0,
      "logps/rejected": -334.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4375,
      "rewards/margins": 14.625,
      "rewards/rejected": -18.125,
      "step": 910
    },
    {
      "epoch": 0.368,
      "grad_norm": 0.00028357064184808366,
      "learning_rate": 4.874074074074073e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.1953125,
      "logps/chosen": -225.0,
      "logps/rejected": -320.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.140625,
      "rewards/margins": 14.0,
      "rewards/rejected": -17.125,
      "step": 920
    },
    {
      "epoch": 0.372,
      "grad_norm": 0.0005193303732379481,
      "learning_rate": 4.866666666666666e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.1875,
      "logps/chosen": -177.0,
      "logps/rejected": -338.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.125,
      "rewards/margins": 14.625,
      "rewards/rejected": -18.75,
      "step": 930
    },
    {
      "epoch": 0.376,
      "grad_norm": 0.000150234868027717,
      "learning_rate": 4.859259259259259e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.28125,
      "logps/chosen": -193.0,
      "logps/rejected": -338.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.1875,
      "rewards/margins": 14.4375,
      "rewards/rejected": -18.625,
      "step": 940
    },
    {
      "epoch": 0.38,
      "grad_norm": 0.000380953426913352,
      "learning_rate": 4.851851851851852e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.1796875,
      "logps/chosen": -178.0,
      "logps/rejected": -336.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.765625,
      "rewards/margins": 14.5,
      "rewards/rejected": -18.25,
      "step": 950
    },
    {
      "epoch": 0.384,
      "grad_norm": 4.938306865294368e-05,
      "learning_rate": 4.844444444444445e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -190.0,
      "logps/rejected": -338.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.734375,
      "rewards/margins": 15.25,
      "rewards/rejected": -19.0,
      "step": 960
    },
    {
      "epoch": 0.388,
      "grad_norm": 0.00010484071430898286,
      "learning_rate": 4.837037037037037e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.2265625,
      "logps/chosen": -162.0,
      "logps/rejected": -334.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9375,
      "rewards/margins": 14.6875,
      "rewards/rejected": -18.625,
      "step": 970
    },
    {
      "epoch": 0.392,
      "grad_norm": 0.005271347918281806,
      "learning_rate": 4.829629629629629e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.125,
      "logps/chosen": -180.0,
      "logps/rejected": -344.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.953125,
      "rewards/margins": 15.5,
      "rewards/rejected": -19.5,
      "step": 980
    },
    {
      "epoch": 0.396,
      "grad_norm": 0.005338567698068648,
      "learning_rate": 4.822222222222222e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.1484375,
      "logps/chosen": -162.0,
      "logps/rejected": -338.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.78125,
      "rewards/margins": 15.1875,
      "rewards/rejected": -19.0,
      "step": 990
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.0002019792966964262,
      "learning_rate": 4.814814814814814e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.21875,
      "logps/chosen": -202.0,
      "logps/rejected": -340.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6875,
      "rewards/margins": 14.6875,
      "rewards/rejected": -18.375,
      "step": 1000
    },
    {
      "epoch": 0.404,
      "grad_norm": 0.0024819050409796563,
      "learning_rate": 4.807407407407407e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.25,
      "logps/chosen": -210.0,
      "logps/rejected": -334.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.03125,
      "rewards/margins": 14.125,
      "rewards/rejected": -18.125,
      "step": 1010
    },
    {
      "epoch": 0.408,
      "grad_norm": 0.004426161255625363,
      "learning_rate": 4.8e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.234375,
      "logps/chosen": -197.0,
      "logps/rejected": -332.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.875,
      "rewards/margins": 14.6875,
      "rewards/rejected": -18.5,
      "step": 1020
    },
    {
      "epoch": 0.412,
      "grad_norm": 7.656828871988746e-05,
      "learning_rate": 4.792592592592592e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.171875,
      "logps/chosen": -180.0,
      "logps/rejected": -332.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.1875,
      "rewards/margins": 14.75,
      "rewards/rejected": -18.875,
      "step": 1030
    },
    {
      "epoch": 0.416,
      "grad_norm": 0.0019573086751353257,
      "learning_rate": 4.785185185185185e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.203125,
      "logps/chosen": -158.0,
      "logps/rejected": -342.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9375,
      "rewards/margins": 15.5,
      "rewards/rejected": -19.5,
      "step": 1040
    },
    {
      "epoch": 0.42,
      "grad_norm": 4.8259920444029365e-05,
      "learning_rate": 4.777777777777778e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -1.2109375,
      "logps/chosen": -170.0,
      "logps/rejected": -328.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.90625,
      "rewards/margins": 14.8125,
      "rewards/rejected": -18.75,
      "step": 1050
    },
    {
      "epoch": 0.424,
      "grad_norm": 0.00018178662409878187,
      "learning_rate": 4.77037037037037e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.234375,
      "logps/chosen": -162.0,
      "logps/rejected": -358.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.40625,
      "rewards/margins": 16.125,
      "rewards/rejected": -20.625,
      "step": 1060
    },
    {
      "epoch": 0.428,
      "grad_norm": 4.7354254983544595e-05,
      "learning_rate": 4.7629629629629626e-07,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.234375,
      "logps/chosen": -201.0,
      "logps/rejected": -336.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.71875,
      "rewards/margins": 15.5,
      "rewards/rejected": -19.25,
      "step": 1070
    },
    {
      "epoch": 0.432,
      "grad_norm": 0.02224061204426968,
      "learning_rate": 4.7555555555555554e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -1.2578125,
      "logps/chosen": -166.0,
      "logps/rejected": -342.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.21875,
      "rewards/margins": 14.875,
      "rewards/rejected": -19.125,
      "step": 1080
    },
    {
      "epoch": 0.436,
      "grad_norm": 2.7277896140218845e-05,
      "learning_rate": 4.7481481481481477e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -167.0,
      "logps/rejected": -348.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.03125,
      "rewards/margins": 16.0,
      "rewards/rejected": -20.0,
      "step": 1090
    },
    {
      "epoch": 0.44,
      "grad_norm": 9.604624714047514e-05,
      "learning_rate": 4.7407407407407405e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.1796875,
      "logps/chosen": -156.0,
      "logps/rejected": -338.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.21875,
      "rewards/margins": 14.6875,
      "rewards/rejected": -18.875,
      "step": 1100
    },
    {
      "epoch": 0.444,
      "grad_norm": 1.1845649860204656e-05,
      "learning_rate": 4.733333333333333e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.2109375,
      "logps/chosen": -197.0,
      "logps/rejected": -344.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.984375,
      "rewards/margins": 16.0,
      "rewards/rejected": -20.0,
      "step": 1110
    },
    {
      "epoch": 0.448,
      "grad_norm": 1.62252224117795e-05,
      "learning_rate": 4.725925925925926e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.0625,
      "logps/chosen": -154.0,
      "logps/rejected": -346.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.984375,
      "rewards/margins": 15.875,
      "rewards/rejected": -19.875,
      "step": 1120
    },
    {
      "epoch": 0.452,
      "grad_norm": 4.455355361229339e-05,
      "learning_rate": 4.7185185185185185e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.1484375,
      "logps/chosen": -159.0,
      "logps/rejected": -354.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.5,
      "rewards/margins": 16.5,
      "rewards/rejected": -21.0,
      "step": 1130
    },
    {
      "epoch": 0.456,
      "grad_norm": 2.104746917887329e-05,
      "learning_rate": 4.711111111111111e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.1640625,
      "logps/chosen": -178.0,
      "logps/rejected": -344.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.125,
      "rewards/margins": 15.6875,
      "rewards/rejected": -19.75,
      "step": 1140
    },
    {
      "epoch": 0.46,
      "grad_norm": 0.0023346881393051864,
      "learning_rate": 4.7037037037037036e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.09375,
      "logps/chosen": -200.0,
      "logps/rejected": -348.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.984375,
      "rewards/margins": 16.0,
      "rewards/rejected": -20.0,
      "step": 1150
    },
    {
      "epoch": 0.464,
      "grad_norm": 4.573526522673348e-05,
      "learning_rate": 4.696296296296296e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.1796875,
      "logps/chosen": -203.0,
      "logps/rejected": -364.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.28125,
      "rewards/margins": 16.75,
      "rewards/rejected": -21.0,
      "step": 1160
    },
    {
      "epoch": 0.468,
      "grad_norm": 0.0017397358028294633,
      "learning_rate": 4.6888888888888887e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.2109375,
      "logps/chosen": -191.0,
      "logps/rejected": -356.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.15625,
      "rewards/margins": 16.25,
      "rewards/rejected": -20.375,
      "step": 1170
    },
    {
      "epoch": 0.472,
      "grad_norm": 8.395298159484358e-06,
      "learning_rate": 4.681481481481481e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.1875,
      "logps/chosen": -159.0,
      "logps/rejected": -358.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.75,
      "rewards/margins": 16.25,
      "rewards/rejected": -21.125,
      "step": 1180
    },
    {
      "epoch": 0.476,
      "grad_norm": 0.05506141110855205,
      "learning_rate": 4.674074074074074e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.125,
      "logps/chosen": -237.0,
      "logps/rejected": -376.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 14.375,
      "rewards/rejected": -22.5,
      "step": 1190
    },
    {
      "epoch": 0.48,
      "grad_norm": 0.071890481294074,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.203125,
      "logps/chosen": -235.0,
      "logps/rejected": -368.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 13.75,
      "rewards/rejected": -22.125,
      "step": 1200
    },
    {
      "epoch": 0.484,
      "grad_norm": 0.000581262022204167,
      "learning_rate": 4.659259259259259e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -1.203125,
      "logps/chosen": -200.0,
      "logps/rejected": -364.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.0,
      "rewards/margins": 14.6875,
      "rewards/rejected": -20.625,
      "step": 1210
    },
    {
      "epoch": 0.488,
      "grad_norm": 0.00034355415301461886,
      "learning_rate": 4.651851851851852e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.98046875,
      "logps/chosen": -166.0,
      "logps/rejected": -346.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.5,
      "rewards/margins": 15.0,
      "rewards/rejected": -19.5,
      "step": 1220
    },
    {
      "epoch": 0.492,
      "grad_norm": 0.00038840134593705767,
      "learning_rate": 4.644444444444444e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.078125,
      "logps/chosen": -176.0,
      "logps/rejected": -330.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.734375,
      "rewards/margins": 15.1875,
      "rewards/rejected": -18.875,
      "step": 1230
    },
    {
      "epoch": 0.496,
      "grad_norm": 0.012096171846181565,
      "learning_rate": 4.637037037037037e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.2265625,
      "logps/chosen": -187.0,
      "logps/rejected": -342.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.921875,
      "rewards/margins": 15.375,
      "rewards/rejected": -19.25,
      "step": 1240
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.0006790523675465874,
      "learning_rate": 4.6296296296296297e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.1953125,
      "logps/chosen": -165.0,
      "logps/rejected": -346.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.4375,
      "rewards/margins": 15.375,
      "rewards/rejected": -19.75,
      "step": 1250
    },
    {
      "epoch": 0.504,
      "grad_norm": 1.692913928196377e-05,
      "learning_rate": 4.622222222222222e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.1015625,
      "logps/chosen": -145.0,
      "logps/rejected": -348.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.953125,
      "rewards/margins": 16.0,
      "rewards/rejected": -20.0,
      "step": 1260
    },
    {
      "epoch": 0.508,
      "grad_norm": 3.7892876748084244e-05,
      "learning_rate": 4.614814814814815e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.1328125,
      "logps/chosen": -167.0,
      "logps/rejected": -344.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9375,
      "rewards/margins": 15.25,
      "rewards/rejected": -19.25,
      "step": 1270
    },
    {
      "epoch": 0.512,
      "grad_norm": 0.00010758469954417227,
      "learning_rate": 4.607407407407407e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.15625,
      "logps/chosen": -168.0,
      "logps/rejected": -346.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.1875,
      "rewards/margins": 16.0,
      "rewards/rejected": -20.25,
      "step": 1280
    },
    {
      "epoch": 0.516,
      "grad_norm": 0.00012170415999081374,
      "learning_rate": 4.6e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -1.0703125,
      "logps/chosen": -208.0,
      "logps/rejected": -364.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.46875,
      "rewards/margins": 18.0,
      "rewards/rejected": -21.5,
      "step": 1290
    },
    {
      "epoch": 0.52,
      "grad_norm": 0.0009618986607260617,
      "learning_rate": 4.592592592592592e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.125,
      "logps/chosen": -157.0,
      "logps/rejected": -354.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.765625,
      "rewards/margins": 16.75,
      "rewards/rejected": -20.5,
      "step": 1300
    },
    {
      "epoch": 0.524,
      "grad_norm": 0.00015988966647762225,
      "learning_rate": 4.5851851851851845e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.1484375,
      "logps/chosen": -168.0,
      "logps/rejected": -346.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9375,
      "rewards/margins": 16.5,
      "rewards/rejected": -20.375,
      "step": 1310
    },
    {
      "epoch": 0.528,
      "grad_norm": 0.000488768594584398,
      "learning_rate": 4.577777777777778e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.1171875,
      "logps/chosen": -181.0,
      "logps/rejected": -378.0,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.09375,
      "rewards/margins": 18.125,
      "rewards/rejected": -23.125,
      "step": 1320
    },
    {
      "epoch": 0.532,
      "grad_norm": 0.003146477751286657,
      "learning_rate": 4.57037037037037e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.95703125,
      "logps/chosen": -209.0,
      "logps/rejected": -362.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.6875,
      "rewards/margins": 14.875,
      "rewards/rejected": -21.5,
      "step": 1330
    },
    {
      "epoch": 0.536,
      "grad_norm": 2.4871965607385428e-05,
      "learning_rate": 4.562962962962963e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.9765625,
      "logps/chosen": -226.0,
      "logps/rejected": -388.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 16.625,
      "rewards/rejected": -24.25,
      "step": 1340
    },
    {
      "epoch": 0.54,
      "grad_norm": 3.056858614427572e-05,
      "learning_rate": 4.555555555555555e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.015625,
      "logps/chosen": -184.0,
      "logps/rejected": -396.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 17.5,
      "rewards/rejected": -25.0,
      "step": 1350
    },
    {
      "epoch": 0.544,
      "grad_norm": 3.791996425849284e-05,
      "learning_rate": 4.548148148148148e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.1015625,
      "logps/chosen": -216.0,
      "logps/rejected": -414.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 18.25,
      "rewards/rejected": -25.75,
      "step": 1360
    },
    {
      "epoch": 0.548,
      "grad_norm": 0.05222770510119527,
      "learning_rate": 4.5407407407407403e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.046875,
      "logps/chosen": -214.0,
      "logps/rejected": -376.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 15.625,
      "rewards/rejected": -23.375,
      "step": 1370
    },
    {
      "epoch": 0.552,
      "grad_norm": 1.1189290782684246e-05,
      "learning_rate": 4.5333333333333326e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.984375,
      "logps/chosen": -186.0,
      "logps/rejected": -408.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 18.25,
      "rewards/rejected": -26.125,
      "step": 1380
    },
    {
      "epoch": 0.556,
      "grad_norm": 0.0025905533349665027,
      "learning_rate": 4.525925925925926e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.1171875,
      "logps/chosen": -237.0,
      "logps/rejected": -390.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 16.875,
      "rewards/rejected": -24.5,
      "step": 1390
    },
    {
      "epoch": 0.56,
      "grad_norm": 5.740129141550371e-06,
      "learning_rate": 4.5185185185185183e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -220.0,
      "logps/rejected": -408.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 18.125,
      "rewards/rejected": -26.125,
      "step": 1400
    },
    {
      "epoch": 0.564,
      "grad_norm": 1.6489535825143758e-06,
      "learning_rate": 4.511111111111111e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.93359375,
      "logps/chosen": -207.0,
      "logps/rejected": -398.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 17.0,
      "rewards/rejected": -24.875,
      "step": 1410
    },
    {
      "epoch": 0.568,
      "grad_norm": 3.3509317944794163e-06,
      "learning_rate": 4.5037037037037034e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.95703125,
      "logps/chosen": -210.0,
      "logps/rejected": -412.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.65625,
      "rewards/margins": 18.5,
      "rewards/rejected": -26.25,
      "step": 1420
    },
    {
      "epoch": 0.572,
      "grad_norm": 1.9004062413634956e-05,
      "learning_rate": 4.496296296296296e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.0703125,
      "logps/chosen": -202.0,
      "logps/rejected": -408.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 19.125,
      "rewards/rejected": -25.875,
      "step": 1430
    },
    {
      "epoch": 0.576,
      "grad_norm": 1.754086380761181e-06,
      "learning_rate": 4.4888888888888885e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.046875,
      "logps/chosen": -208.0,
      "logps/rejected": -408.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.96875,
      "rewards/margins": 18.625,
      "rewards/rejected": -25.625,
      "step": 1440
    },
    {
      "epoch": 0.58,
      "grad_norm": 5.7049561701545345e-05,
      "learning_rate": 4.4814814814814813e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.0703125,
      "logps/chosen": -208.0,
      "logps/rejected": -398.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.90625,
      "rewards/margins": 18.25,
      "rewards/rejected": -25.125,
      "step": 1450
    },
    {
      "epoch": 0.584,
      "grad_norm": 0.00036661830513912103,
      "learning_rate": 4.474074074074074e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -231.0,
      "logps/rejected": -404.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 18.25,
      "rewards/rejected": -25.875,
      "step": 1460
    },
    {
      "epoch": 0.588,
      "grad_norm": 0.00018808789273158842,
      "learning_rate": 4.4666666666666664e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.0546875,
      "logps/chosen": -216.0,
      "logps/rejected": -410.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 18.75,
      "rewards/rejected": -26.25,
      "step": 1470
    },
    {
      "epoch": 0.592,
      "grad_norm": 0.0020062656328177536,
      "learning_rate": 4.459259259259259e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.078125,
      "logps/chosen": -233.0,
      "logps/rejected": -402.0,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 17.625,
      "rewards/rejected": -25.5,
      "step": 1480
    },
    {
      "epoch": 0.596,
      "grad_norm": 1.918816202703681e-05,
      "learning_rate": 4.4518518518518515e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -1.0625,
      "logps/chosen": -206.0,
      "logps/rejected": -410.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 19.125,
      "rewards/rejected": -26.0,
      "step": 1490
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.0007012943278600049,
      "learning_rate": 4.444444444444444e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.0234375,
      "logps/chosen": -209.0,
      "logps/rejected": -410.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 19.25,
      "rewards/rejected": -26.125,
      "step": 1500
    },
    {
      "epoch": 0.604,
      "grad_norm": 3.5433573091884954e-05,
      "learning_rate": 4.4370370370370367e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.078125,
      "logps/chosen": -199.0,
      "logps/rejected": -414.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.46875,
      "rewards/margins": 19.625,
      "rewards/rejected": -27.125,
      "step": 1510
    },
    {
      "epoch": 0.608,
      "grad_norm": 1.0905612922985621e-08,
      "learning_rate": 4.4296296296296295e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.0546875,
      "logps/chosen": -207.0,
      "logps/rejected": -424.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0,
      "rewards/margins": 20.625,
      "rewards/rejected": -27.625,
      "step": 1520
    },
    {
      "epoch": 0.612,
      "grad_norm": 0.0029494430832492425,
      "learning_rate": 4.4222222222222223e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.015625,
      "logps/chosen": -202.0,
      "logps/rejected": -412.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.96875,
      "rewards/margins": 20.25,
      "rewards/rejected": -27.25,
      "step": 1530
    },
    {
      "epoch": 0.616,
      "grad_norm": 6.025397411158037e-05,
      "learning_rate": 4.4148148148148146e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.015625,
      "logps/chosen": -236.0,
      "logps/rejected": -416.0,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 20.0,
      "rewards/rejected": -27.375,
      "step": 1540
    },
    {
      "epoch": 0.62,
      "grad_norm": 0.00019131817973903255,
      "learning_rate": 4.4074074074074074e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -205.0,
      "logps/rejected": -392.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.59375,
      "rewards/margins": 18.875,
      "rewards/rejected": -24.5,
      "step": 1550
    },
    {
      "epoch": 0.624,
      "grad_norm": 2.5953854724402007e-05,
      "learning_rate": 4.3999999999999997e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.96875,
      "logps/chosen": -192.0,
      "logps/rejected": -408.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.4375,
      "rewards/margins": 20.25,
      "rewards/rejected": -26.75,
      "step": 1560
    },
    {
      "epoch": 0.628,
      "grad_norm": 0.00015605357056317355,
      "learning_rate": 4.392592592592592e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.921875,
      "logps/chosen": -223.0,
      "logps/rejected": -404.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.875,
      "rewards/margins": 19.5,
      "rewards/rejected": -26.375,
      "step": 1570
    },
    {
      "epoch": 0.632,
      "grad_norm": 5.897565856295372e-07,
      "learning_rate": 4.3851851851851853e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.94921875,
      "logps/chosen": -213.0,
      "logps/rejected": -406.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.53125,
      "rewards/margins": 19.625,
      "rewards/rejected": -26.125,
      "step": 1580
    },
    {
      "epoch": 0.636,
      "grad_norm": 8.783224186595142e-07,
      "learning_rate": 4.3777777777777776e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -212.0,
      "logps/rejected": -432.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.4375,
      "rewards/margins": 21.75,
      "rewards/rejected": -28.125,
      "step": 1590
    },
    {
      "epoch": 0.64,
      "grad_norm": 5.1301739673906e-05,
      "learning_rate": 4.3703703703703704e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.015625,
      "logps/chosen": -190.0,
      "logps/rejected": -420.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.5625,
      "rewards/margins": 20.5,
      "rewards/rejected": -27.0,
      "step": 1600
    },
    {
      "epoch": 0.644,
      "grad_norm": 0.0004390795232344376,
      "learning_rate": 4.362962962962963e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -205.0,
      "logps/rejected": -420.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.5625,
      "rewards/margins": 21.25,
      "rewards/rejected": -27.75,
      "step": 1610
    },
    {
      "epoch": 0.648,
      "grad_norm": 1.4987721800889963e-07,
      "learning_rate": 4.355555555555555e-07,
      "logits/chosen": -0.9921875,
      "logits/rejected": -1.03125,
      "logps/chosen": -180.0,
      "logps/rejected": -430.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.875,
      "rewards/margins": 20.875,
      "rewards/rejected": -27.625,
      "step": 1620
    },
    {
      "epoch": 0.652,
      "grad_norm": 8.829834092592046e-07,
      "learning_rate": 4.348148148148148e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -243.0,
      "logps/rejected": -426.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.5,
      "rewards/margins": 21.375,
      "rewards/rejected": -27.875,
      "step": 1630
    },
    {
      "epoch": 0.656,
      "grad_norm": 5.534735897979177e-06,
      "learning_rate": 4.34074074074074e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.9765625,
      "logps/chosen": -207.0,
      "logps/rejected": -418.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.59375,
      "rewards/margins": 20.875,
      "rewards/rejected": -27.5,
      "step": 1640
    },
    {
      "epoch": 0.66,
      "grad_norm": 8.646811501688226e-05,
      "learning_rate": 4.3333333333333335e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.88671875,
      "logps/chosen": -218.0,
      "logps/rejected": -400.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.71875,
      "rewards/margins": 18.875,
      "rewards/rejected": -25.625,
      "step": 1650
    },
    {
      "epoch": 0.664,
      "grad_norm": 1.5527938014583734e-05,
      "learning_rate": 4.325925925925926e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.96484375,
      "logps/chosen": -218.0,
      "logps/rejected": -434.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 21.5,
      "rewards/rejected": -28.375,
      "step": 1660
    },
    {
      "epoch": 0.668,
      "grad_norm": 5.343634374764389e-07,
      "learning_rate": 4.3185185185185186e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.9609375,
      "logps/chosen": -227.0,
      "logps/rejected": -428.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.28125,
      "rewards/margins": 22.0,
      "rewards/rejected": -28.25,
      "step": 1670
    },
    {
      "epoch": 0.672,
      "grad_norm": 1.616885129057332e-06,
      "learning_rate": 4.311111111111111e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -1.015625,
      "logps/chosen": -230.0,
      "logps/rejected": -428.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.90625,
      "rewards/margins": 21.125,
      "rewards/rejected": -28.0,
      "step": 1680
    },
    {
      "epoch": 0.676,
      "grad_norm": 1.5955279877732763e-06,
      "learning_rate": 4.303703703703703e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.109375,
      "logps/chosen": -208.0,
      "logps/rejected": -406.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.71875,
      "rewards/margins": 19.0,
      "rewards/rejected": -25.75,
      "step": 1690
    },
    {
      "epoch": 0.68,
      "grad_norm": 4.4773796012019545e-08,
      "learning_rate": 4.296296296296296e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.0390625,
      "logps/chosen": -195.0,
      "logps/rejected": -434.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.125,
      "rewards/margins": 21.5,
      "rewards/rejected": -28.625,
      "step": 1700
    },
    {
      "epoch": 0.684,
      "grad_norm": 7.254706703496731e-06,
      "learning_rate": 4.2888888888888883e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.0625,
      "logps/chosen": -253.0,
      "logps/rejected": -442.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 22.375,
      "rewards/rejected": -29.25,
      "step": 1710
    },
    {
      "epoch": 0.688,
      "grad_norm": 8.192729040329123e-07,
      "learning_rate": 4.2814814814814816e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.99609375,
      "logps/chosen": -209.0,
      "logps/rejected": -412.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.375,
      "rewards/margins": 20.25,
      "rewards/rejected": -26.625,
      "step": 1720
    },
    {
      "epoch": 0.692,
      "grad_norm": 0.0011612789962484303,
      "learning_rate": 4.274074074074074e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.125,
      "logps/chosen": -212.0,
      "logps/rejected": -394.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0625,
      "rewards/margins": 17.875,
      "rewards/rejected": -25.0,
      "step": 1730
    },
    {
      "epoch": 0.696,
      "grad_norm": 7.663501317671998e-05,
      "learning_rate": 4.266666666666667e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.0,
      "logps/chosen": -233.0,
      "logps/rejected": -436.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 21.625,
      "rewards/rejected": -29.125,
      "step": 1740
    },
    {
      "epoch": 0.7,
      "grad_norm": 7.965036994559461e-07,
      "learning_rate": 4.259259259259259e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.0078125,
      "logps/chosen": -212.0,
      "logps/rejected": -426.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 20.0,
      "rewards/rejected": -27.375,
      "step": 1750
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.5265577496841013e-07,
      "learning_rate": 4.2518518518518513e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.046875,
      "logps/chosen": -201.0,
      "logps/rejected": -426.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.15625,
      "rewards/margins": 20.875,
      "rewards/rejected": -28.0,
      "step": 1760
    },
    {
      "epoch": 0.708,
      "grad_norm": 2.0861411581811772e-07,
      "learning_rate": 4.244444444444444e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.953125,
      "logps/chosen": -229.0,
      "logps/rejected": -432.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.15625,
      "rewards/margins": 21.375,
      "rewards/rejected": -28.5,
      "step": 1770
    },
    {
      "epoch": 0.712,
      "grad_norm": 7.557218569176558e-07,
      "learning_rate": 4.237037037037037e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0390625,
      "logps/chosen": -203.0,
      "logps/rejected": -424.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.09375,
      "rewards/margins": 20.625,
      "rewards/rejected": -27.625,
      "step": 1780
    },
    {
      "epoch": 0.716,
      "grad_norm": 0.00073368507559532,
      "learning_rate": 4.22962962962963e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.99609375,
      "logps/chosen": -247.0,
      "logps/rejected": -432.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.65625,
      "rewards/margins": 21.0,
      "rewards/rejected": -28.625,
      "step": 1790
    },
    {
      "epoch": 0.72,
      "grad_norm": 0.013743421484859563,
      "learning_rate": 4.222222222222222e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.92578125,
      "logps/chosen": -272.0,
      "logps/rejected": -440.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.15625,
      "rewards/margins": 22.25,
      "rewards/rejected": -29.375,
      "step": 1800
    },
    {
      "epoch": 0.724,
      "grad_norm": 0.0008714766453308783,
      "learning_rate": 4.2148148148148144e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.046875,
      "logps/chosen": -220.0,
      "logps/rejected": -428.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 20.125,
      "rewards/rejected": -28.125,
      "step": 1810
    },
    {
      "epoch": 0.728,
      "grad_norm": 6.066909621178258e-06,
      "learning_rate": 4.207407407407407e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.984375,
      "logps/chosen": -206.0,
      "logps/rejected": -444.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.15625,
      "rewards/margins": 22.25,
      "rewards/rejected": -29.375,
      "step": 1820
    },
    {
      "epoch": 0.732,
      "grad_norm": 3.4625761134447933e-06,
      "learning_rate": 4.1999999999999995e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.96484375,
      "logps/chosen": -209.0,
      "logps/rejected": -428.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 20.625,
      "rewards/rejected": -28.375,
      "step": 1830
    },
    {
      "epoch": 0.736,
      "grad_norm": 1.6740103616315603e-05,
      "learning_rate": 4.1925925925925923e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.96484375,
      "logps/chosen": -203.0,
      "logps/rejected": -440.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 22.25,
      "rewards/rejected": -29.5,
      "step": 1840
    },
    {
      "epoch": 0.74,
      "grad_norm": 5.96344813308482e-07,
      "learning_rate": 4.185185185185185e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.96875,
      "logps/chosen": -207.0,
      "logps/rejected": -444.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 21.75,
      "rewards/rejected": -29.25,
      "step": 1850
    },
    {
      "epoch": 0.744,
      "grad_norm": 3.2539945307289146e-06,
      "learning_rate": 4.177777777777778e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.0234375,
      "logps/chosen": -220.0,
      "logps/rejected": -422.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.78125,
      "rewards/margins": 21.25,
      "rewards/rejected": -28.0,
      "step": 1860
    },
    {
      "epoch": 0.748,
      "grad_norm": 9.932273295267658e-07,
      "learning_rate": 4.17037037037037e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.9921875,
      "logps/chosen": -226.0,
      "logps/rejected": -444.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0,
      "rewards/margins": 22.0,
      "rewards/rejected": -29.0,
      "step": 1870
    },
    {
      "epoch": 0.752,
      "grad_norm": 4.944865560074764e-08,
      "learning_rate": 4.1629629629629625e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.953125,
      "logps/chosen": -222.0,
      "logps/rejected": -448.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 22.625,
      "rewards/rejected": -30.0,
      "step": 1880
    },
    {
      "epoch": 0.756,
      "grad_norm": 0.0005597952211793904,
      "learning_rate": 4.1555555555555554e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -204.0,
      "logps/rejected": -430.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.46875,
      "rewards/margins": 21.125,
      "rewards/rejected": -28.625,
      "step": 1890
    },
    {
      "epoch": 0.76,
      "grad_norm": 3.865332356729073e-06,
      "learning_rate": 4.1481481481481476e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -205.0,
      "logps/rejected": -448.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 22.25,
      "rewards/rejected": -29.625,
      "step": 1900
    },
    {
      "epoch": 0.764,
      "grad_norm": 8.9464801329807e-07,
      "learning_rate": 4.140740740740741e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.9140625,
      "logps/chosen": -223.0,
      "logps/rejected": -432.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.46875,
      "rewards/margins": 21.125,
      "rewards/rejected": -28.625,
      "step": 1910
    },
    {
      "epoch": 0.768,
      "grad_norm": 1.5987043303558809e-07,
      "learning_rate": 4.1333333333333333e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.90625,
      "logps/chosen": -238.0,
      "logps/rejected": -436.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 21.5,
      "rewards/rejected": -29.0,
      "step": 1920
    },
    {
      "epoch": 0.772,
      "grad_norm": 2.2208044241676946e-07,
      "learning_rate": 4.1259259259259256e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.98046875,
      "logps/chosen": -230.0,
      "logps/rejected": -450.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 22.5,
      "rewards/rejected": -30.0,
      "step": 1930
    },
    {
      "epoch": 0.776,
      "grad_norm": 3.92858929760074e-06,
      "learning_rate": 4.1185185185185184e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.96875,
      "logps/chosen": -219.0,
      "logps/rejected": -452.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 22.0,
      "rewards/rejected": -29.875,
      "step": 1940
    },
    {
      "epoch": 0.78,
      "grad_norm": 3.615820216393288e-05,
      "learning_rate": 4.1111111111111107e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.0859375,
      "logps/chosen": -228.0,
      "logps/rejected": -436.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 21.625,
      "rewards/rejected": -29.0,
      "step": 1950
    },
    {
      "epoch": 0.784,
      "grad_norm": 2.0678014193690642e-09,
      "learning_rate": 4.1037037037037035e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.94921875,
      "logps/chosen": -262.0,
      "logps/rejected": -438.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 21.0,
      "rewards/rejected": -28.75,
      "step": 1960
    },
    {
      "epoch": 0.788,
      "grad_norm": 1.4532205269444855e-07,
      "learning_rate": 4.096296296296296e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.859375,
      "logps/chosen": -218.0,
      "logps/rejected": -438.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0625,
      "rewards/margins": 21.625,
      "rewards/rejected": -28.75,
      "step": 1970
    },
    {
      "epoch": 0.792,
      "grad_norm": 1.310910234061117e-07,
      "learning_rate": 4.088888888888889e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.97265625,
      "logps/chosen": -222.0,
      "logps/rejected": -442.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 21.5,
      "rewards/rejected": -29.375,
      "step": 1980
    },
    {
      "epoch": 0.796,
      "grad_norm": 9.81443594383327e-09,
      "learning_rate": 4.0814814814814814e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.98046875,
      "logps/chosen": -209.0,
      "logps/rejected": -428.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 20.625,
      "rewards/rejected": -28.125,
      "step": 1990
    },
    {
      "epoch": 0.8,
      "grad_norm": 5.3945088384781296e-09,
      "learning_rate": 4.0740740740740737e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.015625,
      "logps/chosen": -226.0,
      "logps/rejected": -410.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.6875,
      "rewards/margins": 19.625,
      "rewards/rejected": -26.25,
      "step": 2000
    },
    {
      "epoch": 0.804,
      "grad_norm": 2.4909181252660647e-08,
      "learning_rate": 4.0666666666666666e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.8203125,
      "logps/chosen": -251.0,
      "logps/rejected": -432.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.9375,
      "rewards/margins": 21.875,
      "rewards/rejected": -28.75,
      "step": 2010
    },
    {
      "epoch": 0.808,
      "grad_norm": 0.0001125423663610549,
      "learning_rate": 4.059259259259259e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.90625,
      "logps/chosen": -192.0,
      "logps/rejected": -456.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.65625,
      "rewards/margins": 23.0,
      "rewards/rejected": -30.625,
      "step": 2020
    },
    {
      "epoch": 0.812,
      "grad_norm": 0.0007472545992506418,
      "learning_rate": 4.0518518518518517e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -218.0,
      "logps/rejected": -450.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 22.125,
      "rewards/rejected": -30.25,
      "step": 2030
    },
    {
      "epoch": 0.816,
      "grad_norm": 3.825133976812771e-05,
      "learning_rate": 4.044444444444444e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.97265625,
      "logps/chosen": -205.0,
      "logps/rejected": -452.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 22.25,
      "rewards/rejected": -30.625,
      "step": 2040
    },
    {
      "epoch": 0.82,
      "grad_norm": 9.664579350659015e-08,
      "learning_rate": 4.0370370370370373e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.93359375,
      "logps/chosen": -218.0,
      "logps/rejected": -464.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 23.5,
      "rewards/rejected": -31.25,
      "step": 2050
    },
    {
      "epoch": 0.824,
      "grad_norm": 2.6099581219657617e-08,
      "learning_rate": 4.0296296296296296e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.953125,
      "logps/chosen": -195.0,
      "logps/rejected": -456.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 22.5,
      "rewards/rejected": -30.25,
      "step": 2060
    },
    {
      "epoch": 0.828,
      "grad_norm": 0.0005870530357334532,
      "learning_rate": 4.022222222222222e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.9453125,
      "logps/chosen": -251.0,
      "logps/rejected": -442.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 22.0,
      "rewards/rejected": -30.125,
      "step": 2070
    },
    {
      "epoch": 0.832,
      "grad_norm": 0.0002877470502796952,
      "learning_rate": 4.0148148148148147e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.89453125,
      "logps/chosen": -231.0,
      "logps/rejected": -436.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 21.75,
      "rewards/rejected": -29.75,
      "step": 2080
    },
    {
      "epoch": 0.836,
      "grad_norm": 3.5319328085446234e-09,
      "learning_rate": 4.007407407407407e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.83203125,
      "logps/chosen": -228.0,
      "logps/rejected": -470.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 24.125,
      "rewards/rejected": -32.25,
      "step": 2090
    },
    {
      "epoch": 0.84,
      "grad_norm": 1.694173482777986e-09,
      "learning_rate": 4e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.84765625,
      "logps/chosen": -229.0,
      "logps/rejected": -464.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 23.0,
      "rewards/rejected": -31.5,
      "step": 2100
    },
    {
      "epoch": 0.844,
      "grad_norm": 3.612489193014174e-07,
      "learning_rate": 3.9925925925925926e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.03125,
      "logps/chosen": -203.0,
      "logps/rejected": -472.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 24.375,
      "rewards/rejected": -32.25,
      "step": 2110
    },
    {
      "epoch": 0.848,
      "grad_norm": 1.859606273362679e-08,
      "learning_rate": 3.985185185185185e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.0,
      "logps/chosen": -255.0,
      "logps/rejected": -448.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 22.0,
      "rewards/rejected": -29.875,
      "step": 2120
    },
    {
      "epoch": 0.852,
      "grad_norm": 3.125938420471554e-06,
      "learning_rate": 3.977777777777778e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.9375,
      "logps/chosen": -245.0,
      "logps/rejected": -454.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 22.5,
      "rewards/rejected": -30.5,
      "step": 2130
    },
    {
      "epoch": 0.856,
      "grad_norm": 2.0125065794811444e-06,
      "learning_rate": 3.97037037037037e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -211.0,
      "logps/rejected": -458.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 23.0,
      "rewards/rejected": -31.0,
      "step": 2140
    },
    {
      "epoch": 0.86,
      "grad_norm": 4.2472348672319745e-06,
      "learning_rate": 3.962962962962963e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.0,
      "logps/chosen": -241.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 23.0,
      "rewards/rejected": -31.25,
      "step": 2150
    },
    {
      "epoch": 0.864,
      "grad_norm": 0.00010153741450393289,
      "learning_rate": 3.955555555555555e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -222.0,
      "logps/rejected": -442.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.125,
      "rewards/margins": 20.5,
      "rewards/rejected": -29.5,
      "step": 2160
    },
    {
      "epoch": 0.868,
      "grad_norm": 0.0014351164928927782,
      "learning_rate": 3.948148148148148e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.91796875,
      "logps/chosen": -216.0,
      "logps/rejected": -448.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 22.0,
      "rewards/rejected": -29.875,
      "step": 2170
    },
    {
      "epoch": 0.872,
      "grad_norm": 4.945802352779656e-06,
      "learning_rate": 3.940740740740741e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.9296875,
      "logps/chosen": -236.0,
      "logps/rejected": -436.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 21.75,
      "rewards/rejected": -29.125,
      "step": 2180
    },
    {
      "epoch": 0.876,
      "grad_norm": 4.589024419811986e-06,
      "learning_rate": 3.933333333333333e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.0078125,
      "logps/chosen": -249.0,
      "logps/rejected": -430.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 20.625,
      "rewards/rejected": -28.75,
      "step": 2190
    },
    {
      "epoch": 0.88,
      "grad_norm": 5.037846188281163e-06,
      "learning_rate": 3.925925925925926e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.91796875,
      "logps/chosen": -227.0,
      "logps/rejected": -458.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 23.75,
      "rewards/rejected": -31.25,
      "step": 2200
    },
    {
      "epoch": 0.884,
      "grad_norm": 9.49640697687011e-05,
      "learning_rate": 3.918518518518518e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.96484375,
      "logps/chosen": -210.0,
      "logps/rejected": -440.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 21.625,
      "rewards/rejected": -29.25,
      "step": 2210
    },
    {
      "epoch": 0.888,
      "grad_norm": 1.3889521712063738e-05,
      "learning_rate": 3.911111111111111e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.078125,
      "logps/chosen": -215.0,
      "logps/rejected": -444.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 21.625,
      "rewards/rejected": -29.375,
      "step": 2220
    },
    {
      "epoch": 0.892,
      "grad_norm": 0.0076974002420864005,
      "learning_rate": 3.9037037037037033e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.9921875,
      "logps/chosen": -202.0,
      "logps/rejected": -458.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 22.75,
      "rewards/rejected": -31.125,
      "step": 2230
    },
    {
      "epoch": 0.896,
      "grad_norm": 3.1996996260824156e-06,
      "learning_rate": 3.8962962962962956e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.91796875,
      "logps/chosen": -203.0,
      "logps/rejected": -476.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 24.375,
      "rewards/rejected": -32.5,
      "step": 2240
    },
    {
      "epoch": 0.9,
      "grad_norm": 0.0023708739950560895,
      "learning_rate": 3.888888888888889e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.98828125,
      "logps/chosen": -228.0,
      "logps/rejected": -468.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 24.0,
      "rewards/rejected": -32.0,
      "step": 2250
    },
    {
      "epoch": 0.904,
      "grad_norm": 1.763696649652061e-06,
      "learning_rate": 3.881481481481481e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.90625,
      "logps/chosen": -230.0,
      "logps/rejected": -452.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 22.875,
      "rewards/rejected": -30.75,
      "step": 2260
    },
    {
      "epoch": 0.908,
      "grad_norm": 1.8800286911737857e-06,
      "learning_rate": 3.874074074074074e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.94140625,
      "logps/chosen": -220.0,
      "logps/rejected": -464.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 23.125,
      "rewards/rejected": -31.5,
      "step": 2270
    },
    {
      "epoch": 0.912,
      "grad_norm": 4.4215122117022634e-06,
      "learning_rate": 3.8666666666666664e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.953125,
      "logps/chosen": -231.0,
      "logps/rejected": -450.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 22.75,
      "rewards/rejected": -30.875,
      "step": 2280
    },
    {
      "epoch": 0.916,
      "grad_norm": 1.5399188014492074e-06,
      "learning_rate": 3.859259259259259e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.9921875,
      "logps/chosen": -214.0,
      "logps/rejected": -468.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 24.125,
      "rewards/rejected": -32.0,
      "step": 2290
    },
    {
      "epoch": 0.92,
      "grad_norm": 0.0007903199795097197,
      "learning_rate": 3.8518518518518515e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.88671875,
      "logps/chosen": -222.0,
      "logps/rejected": -472.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 24.25,
      "rewards/rejected": -32.5,
      "step": 2300
    },
    {
      "epoch": 0.924,
      "grad_norm": 3.270374926924876e-07,
      "learning_rate": 3.8444444444444443e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.015625,
      "logps/chosen": -228.0,
      "logps/rejected": -452.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 23.0,
      "rewards/rejected": -31.375,
      "step": 2310
    },
    {
      "epoch": 0.928,
      "grad_norm": 4.66078854382102e-08,
      "learning_rate": 3.837037037037037e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.94140625,
      "logps/chosen": -237.0,
      "logps/rejected": -476.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 25.0,
      "rewards/rejected": -32.75,
      "step": 2320
    },
    {
      "epoch": 0.932,
      "grad_norm": 7.77102383242385e-06,
      "learning_rate": 3.8296296296296294e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -203.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 23.25,
      "rewards/rejected": -31.375,
      "step": 2330
    },
    {
      "epoch": 0.936,
      "grad_norm": 6.424261430572887e-09,
      "learning_rate": 3.822222222222222e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.890625,
      "logps/chosen": -234.0,
      "logps/rejected": -474.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 24.125,
      "rewards/rejected": -32.75,
      "step": 2340
    },
    {
      "epoch": 0.94,
      "grad_norm": 3.4956686807129392e-09,
      "learning_rate": 3.8148148148148145e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -216.0,
      "logps/rejected": -470.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.625,
      "rewards/margins": 24.625,
      "rewards/rejected": -32.25,
      "step": 2350
    },
    {
      "epoch": 0.944,
      "grad_norm": 8.083586655876509e-07,
      "learning_rate": 3.8074074074074073e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.96484375,
      "logps/chosen": -188.0,
      "logps/rejected": -466.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 23.75,
      "rewards/rejected": -32.0,
      "step": 2360
    },
    {
      "epoch": 0.948,
      "grad_norm": 1.1186604688962057e-07,
      "learning_rate": 3.7999999999999996e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.859375,
      "logps/chosen": -237.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 23.375,
      "rewards/rejected": -31.875,
      "step": 2370
    },
    {
      "epoch": 0.952,
      "grad_norm": 1.2466461196780599e-05,
      "learning_rate": 3.7925925925925924e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -232.0,
      "logps/rejected": -472.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 23.5,
      "rewards/rejected": -32.0,
      "step": 2380
    },
    {
      "epoch": 0.956,
      "grad_norm": 7.446378503095403e-07,
      "learning_rate": 3.785185185185185e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.8515625,
      "logps/chosen": -250.0,
      "logps/rejected": -448.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 22.0,
      "rewards/rejected": -30.5,
      "step": 2390
    },
    {
      "epoch": 0.96,
      "grad_norm": 2.680974931810238e-07,
      "learning_rate": 3.7777777777777775e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.95703125,
      "logps/chosen": -205.0,
      "logps/rejected": -472.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 24.375,
      "rewards/rejected": -32.5,
      "step": 2400
    },
    {
      "epoch": 0.964,
      "grad_norm": 7.157365711518477e-06,
      "learning_rate": 3.7703703703703704e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.91796875,
      "logps/chosen": -216.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 23.25,
      "rewards/rejected": -31.625,
      "step": 2410
    },
    {
      "epoch": 0.968,
      "grad_norm": 2.449062016185779e-06,
      "learning_rate": 3.7629629629629627e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.81640625,
      "logps/chosen": -225.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 26.0,
      "rewards/rejected": -34.0,
      "step": 2420
    },
    {
      "epoch": 0.972,
      "grad_norm": 4.091219674992804e-08,
      "learning_rate": 3.755555555555555e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.9375,
      "logps/chosen": -232.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 23.375,
      "rewards/rejected": -31.375,
      "step": 2430
    },
    {
      "epoch": 0.976,
      "grad_norm": 1.5510994086995653e-07,
      "learning_rate": 3.7481481481481483e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.91796875,
      "logps/chosen": -215.0,
      "logps/rejected": -468.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 24.125,
      "rewards/rejected": -32.25,
      "step": 2440
    },
    {
      "epoch": 0.98,
      "grad_norm": 2.816794029191041e-06,
      "learning_rate": 3.7407407407407406e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -223.0,
      "logps/rejected": -448.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 22.625,
      "rewards/rejected": -30.75,
      "step": 2450
    },
    {
      "epoch": 0.984,
      "grad_norm": 9.639091087264684e-08,
      "learning_rate": 3.7333333333333334e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.91015625,
      "logps/chosen": -227.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 23.0,
      "rewards/rejected": -31.375,
      "step": 2460
    },
    {
      "epoch": 0.988,
      "grad_norm": 1.0153517107833161e-07,
      "learning_rate": 3.7259259259259257e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.7421875,
      "logps/chosen": -236.0,
      "logps/rejected": -468.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0,
      "rewards/margins": 23.375,
      "rewards/rejected": -32.5,
      "step": 2470
    },
    {
      "epoch": 0.992,
      "grad_norm": 3.5428484457306825e-05,
      "learning_rate": 3.7185185185185185e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.73828125,
      "logps/chosen": -242.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 23.125,
      "rewards/rejected": -31.375,
      "step": 2480
    },
    {
      "epoch": 0.996,
      "grad_norm": 1.340730475134467e-08,
      "learning_rate": 3.711111111111111e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.93359375,
      "logps/chosen": -203.0,
      "logps/rejected": -474.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.75,
      "rewards/margins": 23.375,
      "rewards/rejected": -32.0,
      "step": 2490
    },
    {
      "epoch": 1.0,
      "grad_norm": 7.92299532950043e-09,
      "learning_rate": 3.703703703703703e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -208.0,
      "logps/rejected": -474.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 24.75,
      "rewards/rejected": -33.0,
      "step": 2500
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -1.1796875,
      "eval_logits/rejected": -0.87109375,
      "eval_logps/chosen": -226.0,
      "eval_logps/rejected": -476.0,
      "eval_loss": 2.4582709556852933e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -8.3125,
      "eval_rewards/margins": 24.625,
      "eval_rewards/rejected": -33.0,
      "eval_runtime": 43.0305,
      "eval_samples_per_second": 4.648,
      "eval_steps_per_second": 1.162,
      "step": 2500
    },
    {
      "epoch": 1.004,
      "grad_norm": 6.159447638238332e-06,
      "learning_rate": 3.6962962962962965e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.9296875,
      "logps/chosen": -227.0,
      "logps/rejected": -464.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 24.125,
      "rewards/rejected": -32.25,
      "step": 2510
    },
    {
      "epoch": 1.008,
      "grad_norm": 9.148881130903072e-05,
      "learning_rate": 3.688888888888889e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -1.015625,
      "logps/chosen": -203.0,
      "logps/rejected": -472.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 23.875,
      "rewards/rejected": -32.5,
      "step": 2520
    },
    {
      "epoch": 1.012,
      "grad_norm": 1.9839367953381092e-09,
      "learning_rate": 3.6814814814814816e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -202.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 25.375,
      "rewards/rejected": -33.75,
      "step": 2530
    },
    {
      "epoch": 1.016,
      "grad_norm": 6.425517624862737e-11,
      "learning_rate": 3.674074074074074e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.859375,
      "logps/chosen": -227.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.75,
      "step": 2540
    },
    {
      "epoch": 1.02,
      "grad_norm": 2.337342965367701e-06,
      "learning_rate": 3.666666666666666e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -221.0,
      "logps/rejected": -478.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 25.125,
      "rewards/rejected": -33.0,
      "step": 2550
    },
    {
      "epoch": 1.024,
      "grad_norm": 2.464407787182267e-07,
      "learning_rate": 3.659259259259259e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8828125,
      "logps/chosen": -237.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 25.0,
      "rewards/rejected": -33.25,
      "step": 2560
    },
    {
      "epoch": 1.028,
      "grad_norm": 2.2158423667456096e-07,
      "learning_rate": 3.651851851851851e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.9296875,
      "logps/chosen": -230.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 23.125,
      "rewards/rejected": -31.375,
      "step": 2570
    },
    {
      "epoch": 1.032,
      "grad_norm": 5.030546251278047e-06,
      "learning_rate": 3.6444444444444446e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.84765625,
      "logps/chosen": -226.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.6875,
      "rewards/margins": 24.875,
      "rewards/rejected": -33.5,
      "step": 2580
    },
    {
      "epoch": 1.036,
      "grad_norm": 2.3768260005897413e-06,
      "learning_rate": 3.637037037037037e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -254.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.5,
      "step": 2590
    },
    {
      "epoch": 1.04,
      "grad_norm": 4.0317690867634844e-07,
      "learning_rate": 3.6296296296296297e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -220.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 25.25,
      "rewards/rejected": -33.5,
      "step": 2600
    },
    {
      "epoch": 1.044,
      "grad_norm": 2.0308595281220033e-11,
      "learning_rate": 3.622222222222222e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.83203125,
      "logps/chosen": -221.0,
      "logps/rejected": -464.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 24.125,
      "rewards/rejected": -32.25,
      "step": 2610
    },
    {
      "epoch": 1.048,
      "grad_norm": 1.4277073379386617e-06,
      "learning_rate": 3.6148148148148143e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -237.0,
      "logps/rejected": -466.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 23.5,
      "rewards/rejected": -31.625,
      "step": 2620
    },
    {
      "epoch": 1.052,
      "grad_norm": 4.3281613243436663e-07,
      "learning_rate": 3.607407407407407e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.96875,
      "logps/chosen": -241.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 25.125,
      "rewards/rejected": -33.5,
      "step": 2630
    },
    {
      "epoch": 1.056,
      "grad_norm": 1.8659466835604156e-07,
      "learning_rate": 3.6e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.8984375,
      "logps/chosen": -222.0,
      "logps/rejected": -478.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 25.125,
      "rewards/rejected": -33.5,
      "step": 2640
    },
    {
      "epoch": 1.06,
      "grad_norm": 7.487406426145082e-07,
      "learning_rate": 3.592592592592593e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.921875,
      "logps/chosen": -242.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 25.125,
      "rewards/rejected": -33.75,
      "step": 2650
    },
    {
      "epoch": 1.064,
      "grad_norm": 6.587406882556115e-10,
      "learning_rate": 3.585185185185185e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.94140625,
      "logps/chosen": -262.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 25.125,
      "rewards/rejected": -33.5,
      "step": 2660
    },
    {
      "epoch": 1.068,
      "grad_norm": 2.307860603227713e-07,
      "learning_rate": 3.5777777777777773e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.0078125,
      "logps/chosen": -206.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 25.75,
      "rewards/rejected": -34.0,
      "step": 2670
    },
    {
      "epoch": 1.072,
      "grad_norm": 3.7773806277520554e-08,
      "learning_rate": 3.57037037037037e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.9453125,
      "logps/chosen": -231.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 25.375,
      "rewards/rejected": -33.75,
      "step": 2680
    },
    {
      "epoch": 1.076,
      "grad_norm": 7.196910484195392e-07,
      "learning_rate": 3.5629629629629625e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.80859375,
      "logps/chosen": -217.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 25.25,
      "rewards/rejected": -33.5,
      "step": 2690
    },
    {
      "epoch": 1.08,
      "grad_norm": 1.8747039689181967e-06,
      "learning_rate": 3.5555555555555553e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.84765625,
      "logps/chosen": -212.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 26.0,
      "rewards/rejected": -34.25,
      "step": 2700
    },
    {
      "epoch": 1.084,
      "grad_norm": 1.754161127509814e-07,
      "learning_rate": 3.548148148148148e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.91015625,
      "logps/chosen": -248.0,
      "logps/rejected": -478.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 25.5,
      "rewards/rejected": -32.75,
      "step": 2710
    },
    {
      "epoch": 1.088,
      "grad_norm": 8.261648819328004e-06,
      "learning_rate": 3.540740740740741e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.9375,
      "logps/chosen": -196.0,
      "logps/rejected": -432.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 21.75,
      "rewards/rejected": -29.0,
      "step": 2720
    },
    {
      "epoch": 1.092,
      "grad_norm": 2.5254451978802603e-06,
      "learning_rate": 3.533333333333333e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.90625,
      "logps/chosen": -194.0,
      "logps/rejected": -468.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.21875,
      "rewards/margins": 24.375,
      "rewards/rejected": -31.625,
      "step": 2730
    },
    {
      "epoch": 1.096,
      "grad_norm": 9.797131755169137e-08,
      "learning_rate": 3.5259259259259255e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.8125,
      "logps/chosen": -176.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 25.75,
      "rewards/rejected": -33.25,
      "step": 2740
    },
    {
      "epoch": 1.1,
      "grad_norm": 6.441811659374525e-07,
      "learning_rate": 3.5185185185185183e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.765625,
      "logps/chosen": -215.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.75,
      "step": 2750
    },
    {
      "epoch": 1.104,
      "grad_norm": 8.58489322259534e-09,
      "learning_rate": 3.5111111111111106e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.890625,
      "logps/chosen": -226.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.75,
      "step": 2760
    },
    {
      "epoch": 1.108,
      "grad_norm": 2.6465007712824115e-07,
      "learning_rate": 3.503703703703704e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.8828125,
      "logps/chosen": -212.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.25,
      "step": 2770
    },
    {
      "epoch": 1.112,
      "grad_norm": 6.744686769850512e-09,
      "learning_rate": 3.496296296296296e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.96875,
      "logps/chosen": -227.0,
      "logps/rejected": -476.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 25.0,
      "rewards/rejected": -32.75,
      "step": 2780
    },
    {
      "epoch": 1.116,
      "grad_norm": 9.229532323793076e-10,
      "learning_rate": 3.488888888888889e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -218.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 25.75,
      "rewards/rejected": -33.5,
      "step": 2790
    },
    {
      "epoch": 1.12,
      "grad_norm": 4.691228624873704e-09,
      "learning_rate": 3.4814814814814814e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.8515625,
      "logps/chosen": -213.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 25.75,
      "rewards/rejected": -33.75,
      "step": 2800
    },
    {
      "epoch": 1.124,
      "grad_norm": 5.826327440804232e-07,
      "learning_rate": 3.4740740740740737e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -226.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 25.75,
      "rewards/rejected": -33.5,
      "step": 2810
    },
    {
      "epoch": 1.1280000000000001,
      "grad_norm": 3.0477758874775845e-07,
      "learning_rate": 3.4666666666666665e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.9453125,
      "logps/chosen": -205.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 25.375,
      "rewards/rejected": -33.0,
      "step": 2820
    },
    {
      "epoch": 1.1320000000000001,
      "grad_norm": 7.88863227444034e-09,
      "learning_rate": 3.459259259259259e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.7734375,
      "logps/chosen": -229.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.46875,
      "rewards/margins": 25.875,
      "rewards/rejected": -33.5,
      "step": 2830
    },
    {
      "epoch": 1.1360000000000001,
      "grad_norm": 2.6541279705319553e-08,
      "learning_rate": 3.451851851851852e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.94921875,
      "logps/chosen": -228.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 26.75,
      "rewards/rejected": -35.25,
      "step": 2840
    },
    {
      "epoch": 1.1400000000000001,
      "grad_norm": 1.1119174940977656e-07,
      "learning_rate": 3.4444444444444444e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.84375,
      "logps/chosen": -220.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 27.0,
      "rewards/rejected": -35.0,
      "step": 2850
    },
    {
      "epoch": 1.144,
      "grad_norm": 5.316833879884091e-09,
      "learning_rate": 3.4370370370370367e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.89453125,
      "logps/chosen": -246.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0625,
      "rewards/margins": 24.375,
      "rewards/rejected": -33.5,
      "step": 2860
    },
    {
      "epoch": 1.148,
      "grad_norm": 2.286387285828044e-09,
      "learning_rate": 3.4296296296296295e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8828125,
      "logps/chosen": -242.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 26.625,
      "rewards/rejected": -35.0,
      "step": 2870
    },
    {
      "epoch": 1.152,
      "grad_norm": 9.849543209691858e-11,
      "learning_rate": 3.422222222222222e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -204.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 28.0,
      "rewards/rejected": -35.5,
      "step": 2880
    },
    {
      "epoch": 1.156,
      "grad_norm": 2.853985189952659e-05,
      "learning_rate": 3.4148148148148146e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.91796875,
      "logps/chosen": -237.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 26.25,
      "rewards/rejected": -34.75,
      "step": 2890
    },
    {
      "epoch": 1.16,
      "grad_norm": 5.095729355928362e-08,
      "learning_rate": 3.407407407407407e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.953125,
      "logps/chosen": -220.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.5,
      "step": 2900
    },
    {
      "epoch": 1.164,
      "grad_norm": 1.7604303315717182e-08,
      "learning_rate": 3.4000000000000003e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.94140625,
      "logps/chosen": -264.0,
      "logps/rejected": -460.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 22.75,
      "rewards/rejected": -31.25,
      "step": 2910
    },
    {
      "epoch": 1.168,
      "grad_norm": 1.8551116828783325e-10,
      "learning_rate": 3.3925925925925926e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.8046875,
      "logps/chosen": -201.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.75,
      "step": 2920
    },
    {
      "epoch": 1.172,
      "grad_norm": 9.550435876649427e-08,
      "learning_rate": 3.385185185185185e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -225.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 26.5,
      "rewards/rejected": -34.75,
      "step": 2930
    },
    {
      "epoch": 1.176,
      "grad_norm": 1.911403558328408e-09,
      "learning_rate": 3.3777777777777777e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.765625,
      "logps/chosen": -251.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.8125,
      "rewards/margins": 26.5,
      "rewards/rejected": -35.25,
      "step": 2940
    },
    {
      "epoch": 1.18,
      "grad_norm": 1.2497557702001513e-06,
      "learning_rate": 3.37037037037037e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.78125,
      "logps/chosen": -204.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.75,
      "rewards/margins": 26.0,
      "rewards/rejected": -34.75,
      "step": 2950
    },
    {
      "epoch": 1.184,
      "grad_norm": 7.870019435962736e-09,
      "learning_rate": 3.362962962962963e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -236.0,
      "logps/rejected": -472.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 24.75,
      "rewards/rejected": -33.25,
      "step": 2960
    },
    {
      "epoch": 1.188,
      "grad_norm": 3.804007148561365e-05,
      "learning_rate": 3.3555555555555556e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.84375,
      "logps/chosen": -240.0,
      "logps/rejected": -472.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 25.0,
      "rewards/rejected": -33.0,
      "step": 2970
    },
    {
      "epoch": 1.192,
      "grad_norm": 7.796302019940622e-06,
      "learning_rate": 3.348148148148148e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.671875,
      "logps/chosen": -239.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.75,
      "step": 2980
    },
    {
      "epoch": 1.196,
      "grad_norm": 3.3072740523175897e-07,
      "learning_rate": 3.3407407407407407e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.84375,
      "logps/chosen": -199.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 26.5,
      "rewards/rejected": -35.0,
      "step": 2990
    },
    {
      "epoch": 1.2,
      "grad_norm": 1.0872700437184309e-09,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.82421875,
      "logps/chosen": -229.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 27.375,
      "rewards/rejected": -36.0,
      "step": 3000
    },
    {
      "epoch": 1.204,
      "grad_norm": 3.833977927242933e-10,
      "learning_rate": 3.325925925925926e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.76953125,
      "logps/chosen": -215.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 27.0,
      "rewards/rejected": -35.5,
      "step": 3010
    },
    {
      "epoch": 1.208,
      "grad_norm": 0.0010682116556905007,
      "learning_rate": 3.318518518518518e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.90625,
      "logps/chosen": -245.0,
      "logps/rejected": -476.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 25.875,
      "rewards/rejected": -33.5,
      "step": 3020
    },
    {
      "epoch": 1.212,
      "grad_norm": 8.954648712190949e-10,
      "learning_rate": 3.311111111111111e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.80859375,
      "logps/chosen": -226.0,
      "logps/rejected": -470.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 24.5,
      "rewards/rejected": -33.25,
      "step": 3030
    },
    {
      "epoch": 1.216,
      "grad_norm": 2.3159732625481454e-10,
      "learning_rate": 3.303703703703704e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -207.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 25.0,
      "rewards/rejected": -33.25,
      "step": 3040
    },
    {
      "epoch": 1.22,
      "grad_norm": 1.9183888519201133e-08,
      "learning_rate": 3.296296296296296e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.859375,
      "logps/chosen": -221.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 25.625,
      "rewards/rejected": -34.0,
      "step": 3050
    },
    {
      "epoch": 1.224,
      "grad_norm": 8.72340400783045e-11,
      "learning_rate": 3.288888888888889e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -224.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.875,
      "rewards/margins": 26.25,
      "rewards/rejected": -35.25,
      "step": 3060
    },
    {
      "epoch": 1.228,
      "grad_norm": 1.7418487160429303e-07,
      "learning_rate": 3.281481481481481e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.94921875,
      "logps/chosen": -208.0,
      "logps/rejected": -470.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 24.875,
      "rewards/rejected": -32.75,
      "step": 3070
    },
    {
      "epoch": 1.232,
      "grad_norm": 7.409248456865034e-08,
      "learning_rate": 3.274074074074074e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.828125,
      "logps/chosen": -225.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 27.5,
      "rewards/rejected": -36.0,
      "step": 3080
    },
    {
      "epoch": 1.236,
      "grad_norm": 4.464817780216457e-06,
      "learning_rate": 3.2666666666666663e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.7890625,
      "logps/chosen": -215.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 27.5,
      "rewards/rejected": -35.75,
      "step": 3090
    },
    {
      "epoch": 1.24,
      "grad_norm": 4.85632962917935e-10,
      "learning_rate": 3.2592592592592596e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.82421875,
      "logps/chosen": -276.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.125,
      "rewards/margins": 26.125,
      "rewards/rejected": -35.25,
      "step": 3100
    },
    {
      "epoch": 1.244,
      "grad_norm": 8.021746809858664e-10,
      "learning_rate": 3.251851851851852e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -208.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 25.875,
      "rewards/rejected": -34.25,
      "step": 3110
    },
    {
      "epoch": 1.248,
      "grad_norm": 9.239158780024695e-07,
      "learning_rate": 3.244444444444444e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.87890625,
      "logps/chosen": -218.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 25.875,
      "rewards/rejected": -33.75,
      "step": 3120
    },
    {
      "epoch": 1.252,
      "grad_norm": 1.8509339808975565e-07,
      "learning_rate": 3.237037037037037e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.83984375,
      "logps/chosen": -245.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.5,
      "step": 3130
    },
    {
      "epoch": 1.256,
      "grad_norm": 7.590588044400892e-09,
      "learning_rate": 3.2296296296296293e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.82421875,
      "logps/chosen": -204.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.25,
      "step": 3140
    },
    {
      "epoch": 1.26,
      "grad_norm": 8.982597894081323e-05,
      "learning_rate": 3.222222222222222e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.7421875,
      "logps/chosen": -268.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.8125,
      "rewards/margins": 26.125,
      "rewards/rejected": -35.0,
      "step": 3150
    },
    {
      "epoch": 1.264,
      "grad_norm": 5.138107054690088e-08,
      "learning_rate": 3.2148148148148144e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.875,
      "logps/chosen": -220.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 25.0,
      "rewards/rejected": -33.25,
      "step": 3160
    },
    {
      "epoch": 1.268,
      "grad_norm": 4.525713964206703e-08,
      "learning_rate": 3.207407407407407e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -204.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 27.125,
      "rewards/rejected": -34.75,
      "step": 3170
    },
    {
      "epoch": 1.272,
      "grad_norm": 2.979240558266214e-09,
      "learning_rate": 3.2e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.86328125,
      "logps/chosen": -239.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 26.875,
      "rewards/rejected": -35.0,
      "step": 3180
    },
    {
      "epoch": 1.276,
      "grad_norm": 2.3717741541211397e-05,
      "learning_rate": 3.1925925925925924e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.875,
      "logps/chosen": -226.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 25.75,
      "rewards/rejected": -34.0,
      "step": 3190
    },
    {
      "epoch": 1.28,
      "grad_norm": 1.5501268291222191e-09,
      "learning_rate": 3.185185185185185e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.765625,
      "logps/chosen": -201.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 26.5,
      "rewards/rejected": -34.25,
      "step": 3200
    },
    {
      "epoch": 1.284,
      "grad_norm": 8.729825764591862e-10,
      "learning_rate": 3.1777777777777775e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.90625,
      "logps/chosen": -209.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 29.125,
      "rewards/rejected": -37.5,
      "step": 3210
    },
    {
      "epoch": 1.288,
      "grad_norm": 8.414149903868826e-07,
      "learning_rate": 3.1703703703703703e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.78515625,
      "logps/chosen": -227.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.0,
      "step": 3220
    },
    {
      "epoch": 1.292,
      "grad_norm": 1.0765617762820969e-07,
      "learning_rate": 3.1629629629629626e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.8984375,
      "logps/chosen": -225.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 26.25,
      "rewards/rejected": -34.75,
      "step": 3230
    },
    {
      "epoch": 1.296,
      "grad_norm": 8.620345836329598e-07,
      "learning_rate": 3.1555555555555554e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.72265625,
      "logps/chosen": -243.0,
      "logps/rejected": -474.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 26.125,
      "rewards/rejected": -33.5,
      "step": 3240
    },
    {
      "epoch": 1.3,
      "grad_norm": 8.019788289589552e-11,
      "learning_rate": 3.148148148148148e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.93359375,
      "logps/chosen": -241.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.5,
      "step": 3250
    },
    {
      "epoch": 1.304,
      "grad_norm": 7.478607153372065e-06,
      "learning_rate": 3.1407407407407405e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -235.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 26.875,
      "rewards/rejected": -35.0,
      "step": 3260
    },
    {
      "epoch": 1.308,
      "grad_norm": 1.9929708124069567e-09,
      "learning_rate": 3.1333333333333333e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -231.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.8125,
      "rewards/margins": 26.5,
      "rewards/rejected": -35.25,
      "step": 3270
    },
    {
      "epoch": 1.312,
      "grad_norm": 1.8531122977014646e-09,
      "learning_rate": 3.1259259259259256e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.67578125,
      "logps/chosen": -223.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 26.125,
      "rewards/rejected": -34.5,
      "step": 3280
    },
    {
      "epoch": 1.316,
      "grad_norm": 3.83453230184116e-05,
      "learning_rate": 3.118518518518518e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.828125,
      "logps/chosen": -210.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 26.5,
      "rewards/rejected": -34.5,
      "step": 3290
    },
    {
      "epoch": 1.32,
      "grad_norm": 7.361430126859475e-10,
      "learning_rate": 3.111111111111111e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.88671875,
      "logps/chosen": -240.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 27.125,
      "rewards/rejected": -35.5,
      "step": 3300
    },
    {
      "epoch": 1.324,
      "grad_norm": 1.2565783897677425e-08,
      "learning_rate": 3.1037037037037036e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.9765625,
      "logps/chosen": -194.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 26.375,
      "rewards/rejected": -34.25,
      "step": 3310
    },
    {
      "epoch": 1.328,
      "grad_norm": 7.098292965024917e-10,
      "learning_rate": 3.0962962962962964e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.671875,
      "logps/chosen": -243.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 27.875,
      "rewards/rejected": -36.25,
      "step": 3320
    },
    {
      "epoch": 1.332,
      "grad_norm": 9.007648119171881e-10,
      "learning_rate": 3.0888888888888887e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.78515625,
      "logps/chosen": -212.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.25,
      "step": 3330
    },
    {
      "epoch": 1.336,
      "grad_norm": 2.9161557768074476e-08,
      "learning_rate": 3.0814814814814815e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.80078125,
      "logps/chosen": -234.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 27.375,
      "rewards/rejected": -34.75,
      "step": 3340
    },
    {
      "epoch": 1.34,
      "grad_norm": 6.878126593213119e-07,
      "learning_rate": 3.074074074074074e-07,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.71484375,
      "logps/chosen": -211.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 25.5,
      "rewards/rejected": -34.0,
      "step": 3350
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 8.97259407306683e-10,
      "learning_rate": 3.066666666666666e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.875,
      "logps/chosen": -220.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 27.875,
      "rewards/rejected": -35.75,
      "step": 3360
    },
    {
      "epoch": 1.3479999999999999,
      "grad_norm": 1.6101665481450853e-07,
      "learning_rate": 3.0592592592592594e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.8125,
      "logps/chosen": -215.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.75,
      "step": 3370
    },
    {
      "epoch": 1.3519999999999999,
      "grad_norm": 3.2080505573942643e-09,
      "learning_rate": 3.0518518518518517e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.84765625,
      "logps/chosen": -246.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.75,
      "rewards/margins": 26.875,
      "rewards/rejected": -35.75,
      "step": 3380
    },
    {
      "epoch": 1.3559999999999999,
      "grad_norm": 2.552393767431019e-07,
      "learning_rate": 3.0444444444444445e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.8828125,
      "logps/chosen": -205.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.625,
      "rewards/rejected": -37.0,
      "step": 3390
    },
    {
      "epoch": 1.3599999999999999,
      "grad_norm": 1.7561166439312474e-08,
      "learning_rate": 3.037037037037037e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -221.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 25.75,
      "rewards/rejected": -33.5,
      "step": 3400
    },
    {
      "epoch": 1.3639999999999999,
      "grad_norm": 9.92610097510293e-09,
      "learning_rate": 3.0296296296296296e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.796875,
      "logps/chosen": -228.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.75,
      "step": 3410
    },
    {
      "epoch": 1.3679999999999999,
      "grad_norm": 6.956054128011115e-08,
      "learning_rate": 3.022222222222222e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -236.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 27.75,
      "rewards/rejected": -35.75,
      "step": 3420
    },
    {
      "epoch": 1.3719999999999999,
      "grad_norm": 1.8042248787932792e-08,
      "learning_rate": 3.014814814814814e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.796875,
      "logps/chosen": -248.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 25.75,
      "rewards/rejected": -34.5,
      "step": 3430
    },
    {
      "epoch": 1.376,
      "grad_norm": 7.429335267941702e-09,
      "learning_rate": 3.0074074074074076e-07,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.89453125,
      "logps/chosen": -208.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 27.375,
      "rewards/rejected": -35.0,
      "step": 3440
    },
    {
      "epoch": 1.38,
      "grad_norm": 1.1645052037976403e-10,
      "learning_rate": 3e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -251.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 25.0,
      "rewards/rejected": -33.0,
      "step": 3450
    },
    {
      "epoch": 1.384,
      "grad_norm": 8.927767579111681e-10,
      "learning_rate": 2.9925925925925927e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.82421875,
      "logps/chosen": -224.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.25,
      "step": 3460
    },
    {
      "epoch": 1.388,
      "grad_norm": 2.3506556074728165e-07,
      "learning_rate": 2.985185185185185e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.83203125,
      "logps/chosen": -224.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.75,
      "step": 3470
    },
    {
      "epoch": 1.392,
      "grad_norm": 1.5061944473391599e-09,
      "learning_rate": 2.9777777777777773e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.875,
      "logps/chosen": -224.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.5,
      "step": 3480
    },
    {
      "epoch": 1.396,
      "grad_norm": 6.156684631382922e-08,
      "learning_rate": 2.97037037037037e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.6796875,
      "logps/chosen": -202.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 28.75,
      "rewards/rejected": -36.0,
      "step": 3490
    },
    {
      "epoch": 1.4,
      "grad_norm": 1.1615685460973352e-06,
      "learning_rate": 2.962962962962963e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.734375,
      "logps/chosen": -221.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.125,
      "rewards/margins": 29.125,
      "rewards/rejected": -36.25,
      "step": 3500
    },
    {
      "epoch": 1.404,
      "grad_norm": 6.965336929724408e-08,
      "learning_rate": 2.9555555555555557e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -218.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.0,
      "step": 3510
    },
    {
      "epoch": 1.408,
      "grad_norm": 4.27321979494106e-10,
      "learning_rate": 2.948148148148148e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -246.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 26.5,
      "rewards/rejected": -35.0,
      "step": 3520
    },
    {
      "epoch": 1.412,
      "grad_norm": 1.019536374326936e-09,
      "learning_rate": 2.940740740740741e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.875,
      "logps/chosen": -242.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.5,
      "step": 3530
    },
    {
      "epoch": 1.416,
      "grad_norm": 2.2925443310800536e-05,
      "learning_rate": 2.933333333333333e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -226.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 27.875,
      "rewards/rejected": -36.25,
      "step": 3540
    },
    {
      "epoch": 1.42,
      "grad_norm": 1.576973879306525e-08,
      "learning_rate": 2.9259259259259254e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.81640625,
      "logps/chosen": -228.0,
      "logps/rejected": -476.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.75,
      "rewards/margins": 24.75,
      "rewards/rejected": -33.5,
      "step": 3550
    },
    {
      "epoch": 1.424,
      "grad_norm": 4.860182192759929e-05,
      "learning_rate": 2.918518518518518e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -240.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 27.125,
      "rewards/rejected": -35.75,
      "step": 3560
    },
    {
      "epoch": 1.428,
      "grad_norm": 2.7293269332820577e-05,
      "learning_rate": 2.911111111111111e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.875,
      "logps/chosen": -223.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 24.75,
      "rewards/rejected": -33.25,
      "step": 3570
    },
    {
      "epoch": 1.432,
      "grad_norm": 7.881971470398015e-11,
      "learning_rate": 2.903703703703704e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.890625,
      "logps/chosen": -235.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.0,
      "step": 3580
    },
    {
      "epoch": 1.436,
      "grad_norm": 5.21525794696666e-05,
      "learning_rate": 2.896296296296296e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.67578125,
      "logps/chosen": -240.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.25,
      "step": 3590
    },
    {
      "epoch": 1.44,
      "grad_norm": 5.963966621212079e-10,
      "learning_rate": 2.8888888888888885e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -218.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.5,
      "step": 3600
    },
    {
      "epoch": 1.444,
      "grad_norm": 1.4590853685481262e-06,
      "learning_rate": 2.8814814814814813e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -209.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.75,
      "step": 3610
    },
    {
      "epoch": 1.448,
      "grad_norm": 6.072047631109229e-07,
      "learning_rate": 2.8740740740740736e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.81640625,
      "logps/chosen": -247.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 26.625,
      "rewards/rejected": -35.0,
      "step": 3620
    },
    {
      "epoch": 1.452,
      "grad_norm": 1.258385608576159e-05,
      "learning_rate": 2.866666666666667e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.703125,
      "logps/chosen": -208.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 26.875,
      "rewards/rejected": -35.0,
      "step": 3630
    },
    {
      "epoch": 1.456,
      "grad_norm": 5.933988204848187e-10,
      "learning_rate": 2.859259259259259e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.92578125,
      "logps/chosen": -240.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.5,
      "step": 3640
    },
    {
      "epoch": 1.46,
      "grad_norm": 1.3109439875971585e-07,
      "learning_rate": 2.851851851851852e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.75,
      "logps/chosen": -201.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 27.125,
      "rewards/rejected": -35.0,
      "step": 3650
    },
    {
      "epoch": 1.464,
      "grad_norm": 1.071118160472211e-08,
      "learning_rate": 2.8444444444444443e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.71875,
      "logps/chosen": -197.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 25.25,
      "rewards/rejected": -33.25,
      "step": 3660
    },
    {
      "epoch": 1.468,
      "grad_norm": 4.342582560554217e-08,
      "learning_rate": 2.8370370370370366e-07,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8359375,
      "logps/chosen": -182.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.0,
      "step": 3670
    },
    {
      "epoch": 1.472,
      "grad_norm": 7.99408622932329e-10,
      "learning_rate": 2.8296296296296294e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -230.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 26.75,
      "rewards/rejected": -35.0,
      "step": 3680
    },
    {
      "epoch": 1.476,
      "grad_norm": 1.747208724871386e-10,
      "learning_rate": 2.8222222222222217e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.9140625,
      "logps/chosen": -203.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 27.5,
      "rewards/rejected": -35.75,
      "step": 3690
    },
    {
      "epoch": 1.48,
      "grad_norm": 9.360717055821161e-09,
      "learning_rate": 2.814814814814815e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.890625,
      "logps/chosen": -236.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.875,
      "rewards/margins": 26.125,
      "rewards/rejected": -35.25,
      "step": 3700
    },
    {
      "epoch": 1.484,
      "grad_norm": 1.4136345914361872e-08,
      "learning_rate": 2.8074074074074074e-07,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.82421875,
      "logps/chosen": -196.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.5,
      "step": 3710
    },
    {
      "epoch": 1.488,
      "grad_norm": 2.8337961015353785e-07,
      "learning_rate": 2.8e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.85546875,
      "logps/chosen": -252.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 26.0,
      "rewards/rejected": -34.25,
      "step": 3720
    },
    {
      "epoch": 1.492,
      "grad_norm": 1.047405716366011e-09,
      "learning_rate": 2.7925925925925925e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.81640625,
      "logps/chosen": -260.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 28.0,
      "rewards/rejected": -35.75,
      "step": 3730
    },
    {
      "epoch": 1.496,
      "grad_norm": 1.2340409636187705e-07,
      "learning_rate": 2.785185185185185e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -208.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.25,
      "step": 3740
    },
    {
      "epoch": 1.5,
      "grad_norm": 1.030619945809329e-10,
      "learning_rate": 2.7777777777777776e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.86328125,
      "logps/chosen": -214.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 27.125,
      "rewards/rejected": -35.25,
      "step": 3750
    },
    {
      "epoch": 1.504,
      "grad_norm": 6.936345520956418e-08,
      "learning_rate": 2.77037037037037e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.796875,
      "logps/chosen": -213.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 28.75,
      "rewards/rejected": -37.0,
      "step": 3760
    },
    {
      "epoch": 1.508,
      "grad_norm": 3.4528127510090076e-09,
      "learning_rate": 2.762962962962963e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -228.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.25,
      "step": 3770
    },
    {
      "epoch": 1.512,
      "grad_norm": 5.884884093048621e-09,
      "learning_rate": 2.7555555555555555e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -216.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 26.75,
      "rewards/rejected": -35.0,
      "step": 3780
    },
    {
      "epoch": 1.516,
      "grad_norm": 1.7409211568474528e-09,
      "learning_rate": 2.748148148148148e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.7421875,
      "logps/chosen": -229.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.75,
      "step": 3790
    },
    {
      "epoch": 1.52,
      "grad_norm": 5.824026408091721e-08,
      "learning_rate": 2.7407407407407406e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.78515625,
      "logps/chosen": -226.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.25,
      "step": 3800
    },
    {
      "epoch": 1.524,
      "grad_norm": 1.1896278150129928e-09,
      "learning_rate": 2.733333333333333e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -221.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.25,
      "step": 3810
    },
    {
      "epoch": 1.528,
      "grad_norm": 9.60501625332926e-10,
      "learning_rate": 2.725925925925926e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.7734375,
      "logps/chosen": -230.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.5,
      "step": 3820
    },
    {
      "epoch": 1.532,
      "grad_norm": 1.959660289535199e-12,
      "learning_rate": 2.7185185185185186e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.8828125,
      "logps/chosen": -201.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.5,
      "step": 3830
    },
    {
      "epoch": 1.536,
      "grad_norm": 1.157598328234286e-08,
      "learning_rate": 2.7111111111111114e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -216.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 26.875,
      "rewards/rejected": -35.0,
      "step": 3840
    },
    {
      "epoch": 1.54,
      "grad_norm": 1.4665606145857188e-08,
      "learning_rate": 2.7037037037037037e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.83984375,
      "logps/chosen": -224.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.25,
      "step": 3850
    },
    {
      "epoch": 1.544,
      "grad_norm": 1.1921998181915476e-06,
      "learning_rate": 2.696296296296296e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.765625,
      "logps/chosen": -230.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.5,
      "step": 3860
    },
    {
      "epoch": 1.548,
      "grad_norm": 4.74736378407706e-07,
      "learning_rate": 2.688888888888889e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.734375,
      "logps/chosen": -199.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 26.75,
      "rewards/rejected": -34.75,
      "step": 3870
    },
    {
      "epoch": 1.552,
      "grad_norm": 3.5189735897728256e-06,
      "learning_rate": 2.681481481481481e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.71875,
      "logps/chosen": -234.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0625,
      "rewards/margins": 25.125,
      "rewards/rejected": -34.25,
      "step": 3880
    },
    {
      "epoch": 1.556,
      "grad_norm": 0.0006963858731238374,
      "learning_rate": 2.674074074074074e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -227.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.0,
      "step": 3890
    },
    {
      "epoch": 1.56,
      "grad_norm": 7.332680026480988e-10,
      "learning_rate": 2.6666666666666667e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -236.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.0,
      "step": 3900
    },
    {
      "epoch": 1.564,
      "grad_norm": 2.3784918364840258e-11,
      "learning_rate": 2.659259259259259e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.640625,
      "logps/chosen": -228.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.25,
      "step": 3910
    },
    {
      "epoch": 1.568,
      "grad_norm": 4.765630935860906e-08,
      "learning_rate": 2.651851851851852e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.7890625,
      "logps/chosen": -218.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.75,
      "step": 3920
    },
    {
      "epoch": 1.572,
      "grad_norm": 3.3176799214425157e-10,
      "learning_rate": 2.644444444444444e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.828125,
      "logps/chosen": -230.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.0,
      "step": 3930
    },
    {
      "epoch": 1.576,
      "grad_norm": 2.4825777751675217e-10,
      "learning_rate": 2.637037037037037e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.73828125,
      "logps/chosen": -224.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.875,
      "rewards/margins": 27.375,
      "rewards/rejected": -36.25,
      "step": 3940
    },
    {
      "epoch": 1.58,
      "grad_norm": 6.152322269912167e-09,
      "learning_rate": 2.629629629629629e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -223.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 28.375,
      "rewards/rejected": -37.0,
      "step": 3950
    },
    {
      "epoch": 1.584,
      "grad_norm": 5.96518371822665e-11,
      "learning_rate": 2.6222222222222226e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8359375,
      "logps/chosen": -224.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 30.0,
      "rewards/rejected": -38.25,
      "step": 3960
    },
    {
      "epoch": 1.588,
      "grad_norm": 5.797869685574014e-07,
      "learning_rate": 2.614814814814815e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.7734375,
      "logps/chosen": -212.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 27.625,
      "rewards/rejected": -36.0,
      "step": 3970
    },
    {
      "epoch": 1.592,
      "grad_norm": 3.4330375977467506e-10,
      "learning_rate": 2.607407407407407e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.8125,
      "logps/chosen": -209.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.5,
      "step": 3980
    },
    {
      "epoch": 1.596,
      "grad_norm": 7.729025581008743e-06,
      "learning_rate": 2.6e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -228.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.25,
      "step": 3990
    },
    {
      "epoch": 1.6,
      "grad_norm": 2.161980364651152e-08,
      "learning_rate": 2.5925925925925923e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -227.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 27.5,
      "rewards/rejected": -36.0,
      "step": 4000
    },
    {
      "epoch": 1.604,
      "grad_norm": 2.163919714768975e-05,
      "learning_rate": 2.585185185185185e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.89453125,
      "logps/chosen": -222.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.75,
      "rewards/margins": 27.625,
      "rewards/rejected": -36.25,
      "step": 4010
    },
    {
      "epoch": 1.608,
      "grad_norm": 5.672024915231393e-09,
      "learning_rate": 2.5777777777777774e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.7890625,
      "logps/chosen": -237.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.875,
      "rewards/margins": 28.5,
      "rewards/rejected": -37.5,
      "step": 4020
    },
    {
      "epoch": 1.612,
      "grad_norm": 4.0407169710915455e-07,
      "learning_rate": 2.570370370370371e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.6484375,
      "logps/chosen": -210.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.25,
      "step": 4030
    },
    {
      "epoch": 1.616,
      "grad_norm": 7.4432578623265e-08,
      "learning_rate": 2.562962962962963e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.71484375,
      "logps/chosen": -235.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.0,
      "step": 4040
    },
    {
      "epoch": 1.62,
      "grad_norm": 2.2767387664690628e-09,
      "learning_rate": 2.5555555555555553e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -233.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 26.75,
      "rewards/rejected": -35.0,
      "step": 4050
    },
    {
      "epoch": 1.624,
      "grad_norm": 2.5548686037557857e-09,
      "learning_rate": 2.548148148148148e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.8359375,
      "logps/chosen": -229.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.0,
      "step": 4060
    },
    {
      "epoch": 1.6280000000000001,
      "grad_norm": 1.540478963816801e-08,
      "learning_rate": 2.5407407407407404e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.93359375,
      "logps/chosen": -210.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.03125,
      "rewards/margins": 28.0,
      "rewards/rejected": -35.0,
      "step": 4070
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 7.851609270054518e-11,
      "learning_rate": 2.533333333333333e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.8828125,
      "logps/chosen": -191.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.90625,
      "rewards/margins": 26.75,
      "rewards/rejected": -33.75,
      "step": 4080
    },
    {
      "epoch": 1.6360000000000001,
      "grad_norm": 1.067577726299756e-09,
      "learning_rate": 2.5259259259259255e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.81640625,
      "logps/chosen": -260.0,
      "logps/rejected": -470.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 25.0,
      "rewards/rejected": -32.5,
      "step": 4090
    },
    {
      "epoch": 1.6400000000000001,
      "grad_norm": 6.26315135581084e-09,
      "learning_rate": 2.5185185185185184e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.8046875,
      "logps/chosen": -246.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 28.25,
      "rewards/rejected": -35.5,
      "step": 4100
    },
    {
      "epoch": 1.6440000000000001,
      "grad_norm": 1.4772780314710251e-06,
      "learning_rate": 2.511111111111111e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.87890625,
      "logps/chosen": -195.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.25,
      "rewards/margins": 27.25,
      "rewards/rejected": -34.5,
      "step": 4110
    },
    {
      "epoch": 1.6480000000000001,
      "grad_norm": 7.638470509884154e-10,
      "learning_rate": 2.5037037037037035e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.91015625,
      "logps/chosen": -240.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.125,
      "rewards/margins": 26.25,
      "rewards/rejected": -33.25,
      "step": 4120
    },
    {
      "epoch": 1.6520000000000001,
      "grad_norm": 1.367156361350881e-10,
      "learning_rate": 2.4962962962962963e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.8359375,
      "logps/chosen": -190.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 28.75,
      "rewards/rejected": -35.5,
      "step": 4130
    },
    {
      "epoch": 1.6560000000000001,
      "grad_norm": 1.2713429434463427e-10,
      "learning_rate": 2.4888888888888886e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.7578125,
      "logps/chosen": -238.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0625,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.25,
      "step": 4140
    },
    {
      "epoch": 1.6600000000000001,
      "grad_norm": 9.958689840989142e-07,
      "learning_rate": 2.4814814814814814e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.81640625,
      "logps/chosen": -228.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.0,
      "step": 4150
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 7.396069931387133e-10,
      "learning_rate": 2.474074074074074e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.86328125,
      "logps/chosen": -194.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.125,
      "rewards/margins": 29.5,
      "rewards/rejected": -36.5,
      "step": 4160
    },
    {
      "epoch": 1.6680000000000001,
      "grad_norm": 9.998405384224572e-10,
      "learning_rate": 2.4666666666666665e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -223.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 29.5,
      "rewards/rejected": -36.75,
      "step": 4170
    },
    {
      "epoch": 1.6720000000000002,
      "grad_norm": 3.6765765744029065e-10,
      "learning_rate": 2.4592592592592593e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -217.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 28.375,
      "rewards/rejected": -35.75,
      "step": 4180
    },
    {
      "epoch": 1.6760000000000002,
      "grad_norm": 1.378590524799298e-07,
      "learning_rate": 2.4518518518518516e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.86328125,
      "logps/chosen": -203.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0,
      "rewards/margins": 27.25,
      "rewards/rejected": -34.25,
      "step": 4190
    },
    {
      "epoch": 1.6800000000000002,
      "grad_norm": 2.6180535950879933e-08,
      "learning_rate": 2.4444444444444445e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.8125,
      "logps/chosen": -192.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 26.375,
      "rewards/rejected": -33.5,
      "step": 4200
    },
    {
      "epoch": 1.6840000000000002,
      "grad_norm": 2.098210077470771e-07,
      "learning_rate": 2.437037037037037e-07,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.74609375,
      "logps/chosen": -191.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 27.25,
      "rewards/rejected": -34.75,
      "step": 4210
    },
    {
      "epoch": 1.688,
      "grad_norm": 2.675641210941433e-10,
      "learning_rate": 2.4296296296296296e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -201.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0,
      "rewards/margins": 29.125,
      "rewards/rejected": -36.0,
      "step": 4220
    },
    {
      "epoch": 1.692,
      "grad_norm": 3.262272139120333e-06,
      "learning_rate": 2.4222222222222224e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.72265625,
      "logps/chosen": -193.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.5,
      "step": 4230
    },
    {
      "epoch": 1.696,
      "grad_norm": 2.0247164419445643e-10,
      "learning_rate": 2.4148148148148147e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.75,
      "logps/chosen": -222.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.0,
      "step": 4240
    },
    {
      "epoch": 1.7,
      "grad_norm": 1.4172793167089718e-10,
      "learning_rate": 2.407407407407407e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.03125,
      "logps/chosen": -221.0,
      "logps/rejected": -478.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.09375,
      "rewards/margins": 25.875,
      "rewards/rejected": -33.0,
      "step": 4250
    },
    {
      "epoch": 1.704,
      "grad_norm": 8.272345582512161e-08,
      "learning_rate": 2.4e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.84375,
      "logps/chosen": -205.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.65625,
      "rewards/margins": 28.125,
      "rewards/rejected": -34.75,
      "step": 4260
    },
    {
      "epoch": 1.708,
      "grad_norm": 5.903220937093513e-08,
      "learning_rate": 2.3925925925925926e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.85546875,
      "logps/chosen": -244.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.65625,
      "rewards/margins": 27.375,
      "rewards/rejected": -35.0,
      "step": 4270
    },
    {
      "epoch": 1.712,
      "grad_norm": 4.184398216016271e-05,
      "learning_rate": 2.385185185185185e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.79296875,
      "logps/chosen": -207.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.75,
      "step": 4280
    },
    {
      "epoch": 1.716,
      "grad_norm": 1.0116713174179376e-10,
      "learning_rate": 2.3777777777777777e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.82421875,
      "logps/chosen": -217.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 28.125,
      "rewards/rejected": -35.5,
      "step": 4290
    },
    {
      "epoch": 1.72,
      "grad_norm": 1.5892940887926318e-08,
      "learning_rate": 2.3703703703703703e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.69140625,
      "logps/chosen": -228.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.71875,
      "rewards/margins": 29.75,
      "rewards/rejected": -36.5,
      "step": 4300
    },
    {
      "epoch": 1.724,
      "grad_norm": 9.270299335498748e-06,
      "learning_rate": 2.362962962962963e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.75,
      "logps/chosen": -201.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.28125,
      "rewards/margins": 27.625,
      "rewards/rejected": -34.0,
      "step": 4310
    },
    {
      "epoch": 1.728,
      "grad_norm": 2.8701058142452384e-10,
      "learning_rate": 2.3555555555555554e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -214.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.75,
      "rewards/margins": 28.875,
      "rewards/rejected": -35.75,
      "step": 4320
    },
    {
      "epoch": 1.732,
      "grad_norm": 7.68948031943521e-05,
      "learning_rate": 2.348148148148148e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -209.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.09375,
      "rewards/margins": 28.25,
      "rewards/rejected": -35.5,
      "step": 4330
    },
    {
      "epoch": 1.736,
      "grad_norm": 1.8068413281969356e-08,
      "learning_rate": 2.3407407407407405e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.63671875,
      "logps/chosen": -203.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 28.75,
      "rewards/rejected": -36.0,
      "step": 4340
    },
    {
      "epoch": 1.74,
      "grad_norm": 4.924136231256531e-10,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.9296875,
      "logps/chosen": -182.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.96875,
      "rewards/margins": 28.5,
      "rewards/rejected": -35.5,
      "step": 4350
    },
    {
      "epoch": 1.744,
      "grad_norm": 9.128242808297467e-08,
      "learning_rate": 2.325925925925926e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.765625,
      "logps/chosen": -228.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.09375,
      "rewards/margins": 28.875,
      "rewards/rejected": -36.0,
      "step": 4360
    },
    {
      "epoch": 1.748,
      "grad_norm": 2.0559834580387839e-07,
      "learning_rate": 2.3185185185185184e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.85546875,
      "logps/chosen": -183.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.75,
      "rewards/margins": 27.75,
      "rewards/rejected": -34.5,
      "step": 4370
    },
    {
      "epoch": 1.752,
      "grad_norm": 9.902549251016139e-08,
      "learning_rate": 2.311111111111111e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.8828125,
      "logps/chosen": -224.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.0,
      "step": 4380
    },
    {
      "epoch": 1.756,
      "grad_norm": 3.435236509850558e-08,
      "learning_rate": 2.3037037037037035e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.80859375,
      "logps/chosen": -216.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.875,
      "rewards/margins": 27.875,
      "rewards/rejected": -34.75,
      "step": 4390
    },
    {
      "epoch": 1.76,
      "grad_norm": 2.694077410144619e-10,
      "learning_rate": 2.296296296296296e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.7578125,
      "logps/chosen": -230.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.5,
      "step": 4400
    },
    {
      "epoch": 1.764,
      "grad_norm": 1.1759673121688277e-09,
      "learning_rate": 2.288888888888889e-07,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.796875,
      "logps/chosen": -186.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.15625,
      "rewards/margins": 27.75,
      "rewards/rejected": -35.0,
      "step": 4410
    },
    {
      "epoch": 1.768,
      "grad_norm": 7.767985666575122e-08,
      "learning_rate": 2.2814814814814815e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.69921875,
      "logps/chosen": -213.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.75,
      "rewards/margins": 27.0,
      "rewards/rejected": -33.75,
      "step": 4420
    },
    {
      "epoch": 1.772,
      "grad_norm": 1.4610702383518731e-08,
      "learning_rate": 2.274074074074074e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.875,
      "logps/chosen": -206.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.0,
      "step": 4430
    },
    {
      "epoch": 1.776,
      "grad_norm": 5.581212096007815e-10,
      "learning_rate": 2.2666666666666663e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.765625,
      "logps/chosen": -223.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.25,
      "step": 4440
    },
    {
      "epoch": 1.78,
      "grad_norm": 5.634413945648037e-10,
      "learning_rate": 2.2592592592592591e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.87890625,
      "logps/chosen": -234.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.9375,
      "rewards/margins": 27.5,
      "rewards/rejected": -34.5,
      "step": 4450
    },
    {
      "epoch": 1.784,
      "grad_norm": 6.201282260704561e-08,
      "learning_rate": 2.2518518518518517e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.80078125,
      "logps/chosen": -201.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.90625,
      "rewards/margins": 29.75,
      "rewards/rejected": -36.75,
      "step": 4460
    },
    {
      "epoch": 1.788,
      "grad_norm": 1.271422696747978e-07,
      "learning_rate": 2.2444444444444442e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.8984375,
      "logps/chosen": -203.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 28.25,
      "rewards/rejected": -35.0,
      "step": 4470
    },
    {
      "epoch": 1.792,
      "grad_norm": 7.98731418218999e-10,
      "learning_rate": 2.237037037037037e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.81640625,
      "logps/chosen": -195.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 30.75,
      "rewards/rejected": -38.5,
      "step": 4480
    },
    {
      "epoch": 1.796,
      "grad_norm": 5.003608215303399e-07,
      "learning_rate": 2.2296296296296296e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.8203125,
      "logps/chosen": -256.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 27.875,
      "rewards/rejected": -36.0,
      "step": 4490
    },
    {
      "epoch": 1.8,
      "grad_norm": 7.457028729398441e-11,
      "learning_rate": 2.222222222222222e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.7265625,
      "logps/chosen": -210.0,
      "logps/rejected": -544.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.875,
      "rewards/margins": 32.25,
      "rewards/rejected": -39.0,
      "step": 4500
    },
    {
      "epoch": 1.804,
      "grad_norm": 2.3643039147489974e-09,
      "learning_rate": 2.2148148148148147e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -218.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.21875,
      "rewards/margins": 28.25,
      "rewards/rejected": -35.5,
      "step": 4510
    },
    {
      "epoch": 1.808,
      "grad_norm": 1.6453607191114792e-07,
      "learning_rate": 2.2074074074074073e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.828125,
      "logps/chosen": -196.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 29.25,
      "rewards/rejected": -36.5,
      "step": 4520
    },
    {
      "epoch": 1.812,
      "grad_norm": 4.035383033092184e-09,
      "learning_rate": 2.1999999999999998e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.84375,
      "logps/chosen": -228.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.0,
      "step": 4530
    },
    {
      "epoch": 1.8159999999999998,
      "grad_norm": 8.518760906814258e-12,
      "learning_rate": 2.1925925925925927e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -238.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0625,
      "rewards/margins": 29.75,
      "rewards/rejected": -36.75,
      "step": 4540
    },
    {
      "epoch": 1.8199999999999998,
      "grad_norm": 2.6113960003488888e-08,
      "learning_rate": 2.1851851851851852e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -256.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.5,
      "step": 4550
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 7.386606321826423e-10,
      "learning_rate": 2.1777777777777775e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.80859375,
      "logps/chosen": -220.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 29.0,
      "rewards/rejected": -37.0,
      "step": 4560
    },
    {
      "epoch": 1.8279999999999998,
      "grad_norm": 6.758605712701896e-07,
      "learning_rate": 2.17037037037037e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.875,
      "logps/chosen": -241.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 28.25,
      "rewards/rejected": -35.75,
      "step": 4570
    },
    {
      "epoch": 1.8319999999999999,
      "grad_norm": 2.2751820790252694e-07,
      "learning_rate": 2.162962962962963e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.9296875,
      "logps/chosen": -214.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.9375,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.0,
      "step": 4580
    },
    {
      "epoch": 1.8359999999999999,
      "grad_norm": 3.4769844079802506e-10,
      "learning_rate": 2.1555555555555554e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.796875,
      "logps/chosen": -235.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0,
      "rewards/margins": 28.375,
      "rewards/rejected": -35.5,
      "step": 4590
    },
    {
      "epoch": 1.8399999999999999,
      "grad_norm": 8.21281137211122e-08,
      "learning_rate": 2.148148148148148e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.70703125,
      "logps/chosen": -220.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 28.5,
      "rewards/rejected": -35.75,
      "step": 4600
    },
    {
      "epoch": 1.8439999999999999,
      "grad_norm": 2.7745933254219745e-07,
      "learning_rate": 2.1407407407407408e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.83984375,
      "logps/chosen": -250.0,
      "logps/rejected": -480.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 25.25,
      "rewards/rejected": -33.5,
      "step": 4610
    },
    {
      "epoch": 1.8479999999999999,
      "grad_norm": 2.0511084010316565e-08,
      "learning_rate": 2.1333333333333334e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -233.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.0,
      "step": 4620
    },
    {
      "epoch": 1.8519999999999999,
      "grad_norm": 1.6429319637805217e-09,
      "learning_rate": 2.1259259259259257e-07,
      "logits/chosen": -0.94140625,
      "logits/rejected": -0.8828125,
      "logps/chosen": -207.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.25,
      "step": 4630
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 6.089975825978161e-09,
      "learning_rate": 2.1185185185185185e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.84375,
      "logps/chosen": -234.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 30.125,
      "rewards/rejected": -37.5,
      "step": 4640
    },
    {
      "epoch": 1.8599999999999999,
      "grad_norm": 2.2429168426014944e-05,
      "learning_rate": 2.111111111111111e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.8515625,
      "logps/chosen": -207.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.25,
      "rewards/margins": 27.875,
      "rewards/rejected": -35.25,
      "step": 4650
    },
    {
      "epoch": 1.8639999999999999,
      "grad_norm": 9.455862965841545e-08,
      "learning_rate": 2.1037037037037036e-07,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.84765625,
      "logps/chosen": -185.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.96875,
      "rewards/margins": 29.75,
      "rewards/rejected": -36.75,
      "step": 4660
    },
    {
      "epoch": 1.8679999999999999,
      "grad_norm": 1.539806821245256e-05,
      "learning_rate": 2.0962962962962962e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.8203125,
      "logps/chosen": -225.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.0,
      "step": 4670
    },
    {
      "epoch": 1.8719999999999999,
      "grad_norm": 1.9530474606788914e-09,
      "learning_rate": 2.088888888888889e-07,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.8046875,
      "logps/chosen": -191.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.0,
      "step": 4680
    },
    {
      "epoch": 1.876,
      "grad_norm": 1.3623153223224188e-06,
      "learning_rate": 2.0814814814814813e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -242.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.5,
      "step": 4690
    },
    {
      "epoch": 1.88,
      "grad_norm": 7.600527341916382e-07,
      "learning_rate": 2.0740740740740738e-07,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -200.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 28.125,
      "rewards/rejected": -35.5,
      "step": 4700
    },
    {
      "epoch": 1.884,
      "grad_norm": 5.4333386183299875e-09,
      "learning_rate": 2.0666666666666666e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.68359375,
      "logps/chosen": -217.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.25,
      "rewards/margins": 30.5,
      "rewards/rejected": -37.75,
      "step": 4710
    },
    {
      "epoch": 1.888,
      "grad_norm": 4.431693001788134e-07,
      "learning_rate": 2.0592592592592592e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.9140625,
      "logps/chosen": -238.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 27.125,
      "rewards/rejected": -34.5,
      "step": 4720
    },
    {
      "epoch": 1.892,
      "grad_norm": 1.2574356942872992e-07,
      "learning_rate": 2.0518518518518518e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.81640625,
      "logps/chosen": -241.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.0,
      "step": 4730
    },
    {
      "epoch": 1.896,
      "grad_norm": 3.1919188144199366e-06,
      "learning_rate": 2.0444444444444446e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.8125,
      "logps/chosen": -229.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 27.875,
      "rewards/rejected": -35.25,
      "step": 4740
    },
    {
      "epoch": 1.9,
      "grad_norm": 3.1264631773826693e-12,
      "learning_rate": 2.0370370370370369e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.79296875,
      "logps/chosen": -211.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 27.75,
      "rewards/rejected": -35.25,
      "step": 4750
    },
    {
      "epoch": 1.904,
      "grad_norm": 1.2032249040267508e-07,
      "learning_rate": 2.0296296296296294e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.83203125,
      "logps/chosen": -217.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.0,
      "step": 4760
    },
    {
      "epoch": 1.908,
      "grad_norm": 4.893972679869363e-09,
      "learning_rate": 2.022222222222222e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.765625,
      "logps/chosen": -234.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 27.375,
      "rewards/rejected": -34.75,
      "step": 4770
    },
    {
      "epoch": 1.912,
      "grad_norm": 3.5275909599096966e-09,
      "learning_rate": 2.0148148148148148e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -236.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 26.875,
      "rewards/rejected": -35.0,
      "step": 4780
    },
    {
      "epoch": 1.916,
      "grad_norm": 1.024092426107644e-06,
      "learning_rate": 2.0074074074074074e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.78125,
      "logps/chosen": -214.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 28.25,
      "rewards/rejected": -35.75,
      "step": 4790
    },
    {
      "epoch": 1.92,
      "grad_norm": 7.078393137058789e-11,
      "learning_rate": 2e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.890625,
      "logps/chosen": -186.0,
      "logps/rejected": -478.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.28125,
      "rewards/margins": 26.75,
      "rewards/rejected": -34.0,
      "step": 4800
    },
    {
      "epoch": 1.924,
      "grad_norm": 8.912559128706684e-11,
      "learning_rate": 1.9925925925925925e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.8515625,
      "logps/chosen": -217.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0625,
      "rewards/margins": 27.125,
      "rewards/rejected": -34.25,
      "step": 4810
    },
    {
      "epoch": 1.928,
      "grad_norm": 2.1272741980306116e-11,
      "learning_rate": 1.985185185185185e-07,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -203.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.25,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.25,
      "step": 4820
    },
    {
      "epoch": 1.932,
      "grad_norm": 3.0222376076353466e-09,
      "learning_rate": 1.9777777777777776e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.73828125,
      "logps/chosen": -217.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 28.0,
      "rewards/rejected": -35.25,
      "step": 4830
    },
    {
      "epoch": 1.936,
      "grad_norm": 0.0010094638356249993,
      "learning_rate": 1.9703703703703704e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.77734375,
      "logps/chosen": -236.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 28.5,
      "rewards/rejected": -35.75,
      "step": 4840
    },
    {
      "epoch": 1.94,
      "grad_norm": 3.1358584106348275e-06,
      "learning_rate": 1.962962962962963e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.828125,
      "logps/chosen": -228.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.0,
      "step": 4850
    },
    {
      "epoch": 1.944,
      "grad_norm": 8.558898635693941e-07,
      "learning_rate": 1.9555555555555555e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.73828125,
      "logps/chosen": -210.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.75,
      "step": 4860
    },
    {
      "epoch": 1.948,
      "grad_norm": 8.497784169846918e-08,
      "learning_rate": 1.9481481481481478e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.80078125,
      "logps/chosen": -203.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.8125,
      "rewards/margins": 29.0,
      "rewards/rejected": -35.75,
      "step": 4870
    },
    {
      "epoch": 1.952,
      "grad_norm": 2.561184564222696e-07,
      "learning_rate": 1.9407407407407406e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.87109375,
      "logps/chosen": -224.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.96875,
      "rewards/margins": 28.875,
      "rewards/rejected": -35.75,
      "step": 4880
    },
    {
      "epoch": 1.956,
      "grad_norm": 3.2623823948010295e-06,
      "learning_rate": 1.9333333333333332e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.76953125,
      "logps/chosen": -225.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0,
      "rewards/margins": 28.625,
      "rewards/rejected": -35.75,
      "step": 4890
    },
    {
      "epoch": 1.96,
      "grad_norm": 1.8162329717643264e-08,
      "learning_rate": 1.9259259259259257e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.84375,
      "logps/chosen": -247.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 27.5,
      "rewards/rejected": -35.0,
      "step": 4900
    },
    {
      "epoch": 1.964,
      "grad_norm": 2.283009497671728e-09,
      "learning_rate": 1.9185185185185186e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -221.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.25,
      "rewards/margins": 28.375,
      "rewards/rejected": -35.75,
      "step": 4910
    },
    {
      "epoch": 1.968,
      "grad_norm": 8.040869716303252e-11,
      "learning_rate": 1.911111111111111e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.734375,
      "logps/chosen": -236.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 30.5,
      "rewards/rejected": -38.0,
      "step": 4920
    },
    {
      "epoch": 1.972,
      "grad_norm": 1.3741074849930546e-06,
      "learning_rate": 1.9037037037037037e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.84375,
      "logps/chosen": -226.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 27.375,
      "rewards/rejected": -34.75,
      "step": 4930
    },
    {
      "epoch": 1.976,
      "grad_norm": 8.549134157611614e-10,
      "learning_rate": 1.8962962962962962e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.6953125,
      "logps/chosen": -214.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 30.75,
      "rewards/rejected": -38.0,
      "step": 4940
    },
    {
      "epoch": 1.98,
      "grad_norm": 4.0706880470793766e-08,
      "learning_rate": 1.8888888888888888e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.79296875,
      "logps/chosen": -196.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.03125,
      "rewards/margins": 30.25,
      "rewards/rejected": -37.25,
      "step": 4950
    },
    {
      "epoch": 1.984,
      "grad_norm": 3.071947586807495e-05,
      "learning_rate": 1.8814814814814813e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.76171875,
      "logps/chosen": -210.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.78125,
      "rewards/margins": 29.25,
      "rewards/rejected": -36.0,
      "step": 4960
    },
    {
      "epoch": 1.988,
      "grad_norm": 4.611245239874446e-08,
      "learning_rate": 1.8740740740740742e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -184.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.75,
      "rewards/margins": 30.625,
      "rewards/rejected": -37.5,
      "step": 4970
    },
    {
      "epoch": 1.992,
      "grad_norm": 8.958458297908253e-10,
      "learning_rate": 1.8666666666666667e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -167.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.5,
      "rewards/margins": 28.875,
      "rewards/rejected": -35.5,
      "step": 4980
    },
    {
      "epoch": 1.996,
      "grad_norm": 3.903482995181629e-10,
      "learning_rate": 1.8592592592592593e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.7578125,
      "logps/chosen": -210.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.0,
      "step": 4990
    },
    {
      "epoch": 2.0,
      "grad_norm": 9.941569368265377e-10,
      "learning_rate": 1.8518518518518516e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.74609375,
      "logps/chosen": -227.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 30.0,
      "rewards/rejected": -37.5,
      "step": 5000
    },
    {
      "epoch": 2.0,
      "eval_logits/chosen": -1.125,
      "eval_logits/rejected": -0.75390625,
      "eval_logps/chosen": -215.0,
      "eval_logps/rejected": -502.0,
      "eval_loss": 4.101671891021397e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -7.15625,
      "eval_rewards/margins": 28.375,
      "eval_rewards/rejected": -35.5,
      "eval_runtime": 43.7757,
      "eval_samples_per_second": 4.569,
      "eval_steps_per_second": 1.142,
      "step": 5000
    },
    {
      "epoch": 2.004,
      "grad_norm": 2.8555980911455267e-08,
      "learning_rate": 1.8444444444444444e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.8125,
      "logps/chosen": -218.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0625,
      "rewards/margins": 31.125,
      "rewards/rejected": -38.25,
      "step": 5010
    },
    {
      "epoch": 2.008,
      "grad_norm": 2.961339510551853e-09,
      "learning_rate": 1.837037037037037e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.68359375,
      "logps/chosen": -238.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 30.75,
      "rewards/rejected": -38.25,
      "step": 5020
    },
    {
      "epoch": 2.012,
      "grad_norm": 3.9556172252619e-10,
      "learning_rate": 1.8296296296296295e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.8125,
      "logps/chosen": -210.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.84375,
      "rewards/margins": 29.875,
      "rewards/rejected": -36.75,
      "step": 5030
    },
    {
      "epoch": 2.016,
      "grad_norm": 2.73836246046063e-07,
      "learning_rate": 1.8222222222222223e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.69140625,
      "logps/chosen": -253.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.21875,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.0,
      "step": 5040
    },
    {
      "epoch": 2.02,
      "grad_norm": 1.0253289317884494e-07,
      "learning_rate": 1.8148148148148149e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.76171875,
      "logps/chosen": -216.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 27.0,
      "rewards/rejected": -35.25,
      "step": 5050
    },
    {
      "epoch": 2.024,
      "grad_norm": 5.462647681920073e-09,
      "learning_rate": 1.8074074074074072e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.796875,
      "logps/chosen": -199.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.0,
      "step": 5060
    },
    {
      "epoch": 2.028,
      "grad_norm": 1.6430461806508472e-06,
      "learning_rate": 1.8e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.78515625,
      "logps/chosen": -226.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.34375,
      "rewards/margins": 27.875,
      "rewards/rejected": -35.25,
      "step": 5070
    },
    {
      "epoch": 2.032,
      "grad_norm": 1.0644077796143842e-09,
      "learning_rate": 1.7925925925925925e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.8125,
      "logps/chosen": -201.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 26.75,
      "rewards/rejected": -34.25,
      "step": 5080
    },
    {
      "epoch": 2.036,
      "grad_norm": 3.162425161285629e-08,
      "learning_rate": 1.785185185185185e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.73828125,
      "logps/chosen": -224.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.1875,
      "rewards/margins": 27.125,
      "rewards/rejected": -34.25,
      "step": 5090
    },
    {
      "epoch": 2.04,
      "grad_norm": 7.593347324743958e-08,
      "learning_rate": 1.7777777777777776e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.703125,
      "logps/chosen": -209.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.09375,
      "rewards/margins": 28.75,
      "rewards/rejected": -35.75,
      "step": 5100
    },
    {
      "epoch": 2.044,
      "grad_norm": 5.37315035026481e-10,
      "learning_rate": 1.7703703703703705e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.7578125,
      "logps/chosen": -219.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.25,
      "rewards/margins": 29.125,
      "rewards/rejected": -36.5,
      "step": 5110
    },
    {
      "epoch": 2.048,
      "grad_norm": 7.728146344237148e-11,
      "learning_rate": 1.7629629629629627e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -224.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.0,
      "step": 5120
    },
    {
      "epoch": 2.052,
      "grad_norm": 2.2346524679167233e-09,
      "learning_rate": 1.7555555555555553e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.7890625,
      "logps/chosen": -220.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.625,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.25,
      "step": 5130
    },
    {
      "epoch": 2.056,
      "grad_norm": 1.0866349467254254e-07,
      "learning_rate": 1.748148148148148e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.66015625,
      "logps/chosen": -216.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.125,
      "rewards/margins": 29.125,
      "rewards/rejected": -36.25,
      "step": 5140
    },
    {
      "epoch": 2.06,
      "grad_norm": 5.567418618146607e-08,
      "learning_rate": 1.7407407407407407e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.8125,
      "logps/chosen": -209.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 28.875,
      "rewards/rejected": -36.5,
      "step": 5150
    },
    {
      "epoch": 2.064,
      "grad_norm": 3.1764770826173755e-08,
      "learning_rate": 1.7333333333333332e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.61328125,
      "logps/chosen": -192.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.5,
      "step": 5160
    },
    {
      "epoch": 2.068,
      "grad_norm": 6.612097168512467e-07,
      "learning_rate": 1.725925925925926e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.7578125,
      "logps/chosen": -232.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.75,
      "rewards/rejected": -37.0,
      "step": 5170
    },
    {
      "epoch": 2.072,
      "grad_norm": 2.127614696315248e-10,
      "learning_rate": 1.7185185185185183e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.7421875,
      "logps/chosen": -233.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 30.375,
      "rewards/rejected": -38.25,
      "step": 5180
    },
    {
      "epoch": 2.076,
      "grad_norm": 1.546985362729845e-05,
      "learning_rate": 1.711111111111111e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.85546875,
      "logps/chosen": -194.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 28.75,
      "rewards/rejected": -36.25,
      "step": 5190
    },
    {
      "epoch": 2.08,
      "grad_norm": 2.4964506423618008e-09,
      "learning_rate": 1.7037037037037035e-07,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.80078125,
      "logps/chosen": -196.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 28.0,
      "rewards/rejected": -35.75,
      "step": 5200
    },
    {
      "epoch": 2.084,
      "grad_norm": 3.0868084287565547e-09,
      "learning_rate": 1.6962962962962963e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.88671875,
      "logps/chosen": -242.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 29.625,
      "rewards/rejected": -38.25,
      "step": 5210
    },
    {
      "epoch": 2.088,
      "grad_norm": 9.210891203316807e-07,
      "learning_rate": 1.6888888888888888e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -239.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.75,
      "step": 5220
    },
    {
      "epoch": 2.092,
      "grad_norm": 2.1607140158154548e-09,
      "learning_rate": 1.6814814814814814e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.6953125,
      "logps/chosen": -231.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.25,
      "step": 5230
    },
    {
      "epoch": 2.096,
      "grad_norm": 1.342207278910772e-08,
      "learning_rate": 1.674074074074074e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.74609375,
      "logps/chosen": -206.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 27.75,
      "rewards/rejected": -35.25,
      "step": 5240
    },
    {
      "epoch": 2.1,
      "grad_norm": 1.2059219266767773e-08,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.7421875,
      "logps/chosen": -219.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.0,
      "step": 5250
    },
    {
      "epoch": 2.104,
      "grad_norm": 4.154014746722981e-08,
      "learning_rate": 1.659259259259259e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.83203125,
      "logps/chosen": -227.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.75,
      "step": 5260
    },
    {
      "epoch": 2.108,
      "grad_norm": 2.2510891193418538e-10,
      "learning_rate": 1.651851851851852e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.8359375,
      "logps/chosen": -192.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 28.875,
      "rewards/rejected": -36.5,
      "step": 5270
    },
    {
      "epoch": 2.112,
      "grad_norm": 1.1844815915370776e-07,
      "learning_rate": 1.6444444444444444e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.78125,
      "logps/chosen": -216.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 27.75,
      "rewards/rejected": -35.5,
      "step": 5280
    },
    {
      "epoch": 2.116,
      "grad_norm": 2.8377184050949665e-06,
      "learning_rate": 1.637037037037037e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.76171875,
      "logps/chosen": -219.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 28.375,
      "rewards/rejected": -35.75,
      "step": 5290
    },
    {
      "epoch": 2.12,
      "grad_norm": 6.25292905515715e-10,
      "learning_rate": 1.6296296296296298e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.72265625,
      "logps/chosen": -226.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.25,
      "step": 5300
    },
    {
      "epoch": 2.124,
      "grad_norm": 1.4335854309262707e-07,
      "learning_rate": 1.622222222222222e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.71484375,
      "logps/chosen": -244.0,
      "logps/rejected": -540.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 30.75,
      "rewards/rejected": -39.0,
      "step": 5310
    },
    {
      "epoch": 2.128,
      "grad_norm": 3.461858201893177e-07,
      "learning_rate": 1.6148148148148147e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.7578125,
      "logps/chosen": -224.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 31.25,
      "rewards/rejected": -38.75,
      "step": 5320
    },
    {
      "epoch": 2.132,
      "grad_norm": 5.106858393953262e-12,
      "learning_rate": 1.6074074074074072e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.7265625,
      "logps/chosen": -232.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.625,
      "rewards/margins": 30.25,
      "rewards/rejected": -38.0,
      "step": 5330
    },
    {
      "epoch": 2.136,
      "grad_norm": 1.1237852641656333e-06,
      "learning_rate": 1.6e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.7734375,
      "logps/chosen": -221.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 30.0,
      "rewards/rejected": -38.0,
      "step": 5340
    },
    {
      "epoch": 2.14,
      "grad_norm": 1.6182141972429176e-05,
      "learning_rate": 1.5925925925925926e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.84765625,
      "logps/chosen": -231.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 27.75,
      "rewards/rejected": -36.0,
      "step": 5350
    },
    {
      "epoch": 2.144,
      "grad_norm": 1.558209731047582e-05,
      "learning_rate": 1.5851851851851851e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.765625,
      "logps/chosen": -218.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.28125,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.25,
      "step": 5360
    },
    {
      "epoch": 2.148,
      "grad_norm": 7.143399142227032e-07,
      "learning_rate": 1.5777777777777777e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.828125,
      "logps/chosen": -252.0,
      "logps/rejected": -486.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 26.25,
      "rewards/rejected": -34.25,
      "step": 5370
    },
    {
      "epoch": 2.152,
      "grad_norm": 2.7160289262078327e-06,
      "learning_rate": 1.5703703703703703e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.73828125,
      "logps/chosen": -189.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 29.25,
      "rewards/rejected": -36.75,
      "step": 5380
    },
    {
      "epoch": 2.156,
      "grad_norm": 5.7205413603860956e-08,
      "learning_rate": 1.5629629629629628e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.84375,
      "logps/chosen": -187.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.625,
      "rewards/margins": 28.75,
      "rewards/rejected": -36.25,
      "step": 5390
    },
    {
      "epoch": 2.16,
      "grad_norm": 9.653883904565312e-09,
      "learning_rate": 1.5555555555555556e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.76953125,
      "logps/chosen": -208.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 27.75,
      "rewards/rejected": -35.25,
      "step": 5400
    },
    {
      "epoch": 2.164,
      "grad_norm": 5.464763269052405e-10,
      "learning_rate": 1.5481481481481482e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -206.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.375,
      "rewards/margins": 29.25,
      "rewards/rejected": -36.5,
      "step": 5410
    },
    {
      "epoch": 2.168,
      "grad_norm": 4.1701075118280667e-10,
      "learning_rate": 1.5407407407407407e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.91796875,
      "logps/chosen": -210.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.03125,
      "rewards/margins": 28.0,
      "rewards/rejected": -35.0,
      "step": 5420
    },
    {
      "epoch": 2.172,
      "grad_norm": 2.509355320657385e-08,
      "learning_rate": 1.533333333333333e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.80078125,
      "logps/chosen": -225.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 27.0,
      "rewards/rejected": -35.5,
      "step": 5430
    },
    {
      "epoch": 2.176,
      "grad_norm": 4.837828149330957e-10,
      "learning_rate": 1.5259259259259259e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.81640625,
      "logps/chosen": -237.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.21875,
      "rewards/margins": 30.875,
      "rewards/rejected": -38.0,
      "step": 5440
    },
    {
      "epoch": 2.18,
      "grad_norm": 1.0861505306887942e-10,
      "learning_rate": 1.5185185185185184e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.73828125,
      "logps/chosen": -222.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 5450
    },
    {
      "epoch": 2.184,
      "grad_norm": 1.7968550559383197e-07,
      "learning_rate": 1.511111111111111e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.74609375,
      "logps/chosen": -198.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 26.0,
      "rewards/rejected": -34.0,
      "step": 5460
    },
    {
      "epoch": 2.188,
      "grad_norm": 1.6978675365538846e-10,
      "learning_rate": 1.5037037037037038e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.73046875,
      "logps/chosen": -212.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 28.75,
      "rewards/rejected": -36.25,
      "step": 5470
    },
    {
      "epoch": 2.192,
      "grad_norm": 1.6520691307538226e-11,
      "learning_rate": 1.4962962962962963e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -202.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.4375,
      "rewards/margins": 30.75,
      "rewards/rejected": -38.25,
      "step": 5480
    },
    {
      "epoch": 2.196,
      "grad_norm": 1.6485818667707375e-05,
      "learning_rate": 1.4888888888888886e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.9140625,
      "logps/chosen": -222.0,
      "logps/rejected": -484.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 25.75,
      "rewards/rejected": -33.5,
      "step": 5490
    },
    {
      "epoch": 2.2,
      "grad_norm": 2.527048699747057e-08,
      "learning_rate": 1.4814814814814815e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.6640625,
      "logps/chosen": -238.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 28.25,
      "rewards/rejected": -35.75,
      "step": 5500
    },
    {
      "epoch": 2.204,
      "grad_norm": 5.59489270339163e-07,
      "learning_rate": 1.474074074074074e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.70703125,
      "logps/chosen": -233.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.75,
      "step": 5510
    },
    {
      "epoch": 2.208,
      "grad_norm": 1.3749315614813025e-09,
      "learning_rate": 1.4666666666666666e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.7109375,
      "logps/chosen": -205.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.28125,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.25,
      "step": 5520
    },
    {
      "epoch": 2.212,
      "grad_norm": 7.440737114216714e-12,
      "learning_rate": 1.459259259259259e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.6328125,
      "logps/chosen": -217.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 29.125,
      "rewards/rejected": -36.75,
      "step": 5530
    },
    {
      "epoch": 2.216,
      "grad_norm": 4.5237784744826835e-10,
      "learning_rate": 1.451851851851852e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.73046875,
      "logps/chosen": -198.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.625,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.25,
      "step": 5540
    },
    {
      "epoch": 2.22,
      "grad_norm": 2.233040290544414e-07,
      "learning_rate": 1.4444444444444442e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.8046875,
      "logps/chosen": -264.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 26.875,
      "rewards/rejected": -35.5,
      "step": 5550
    },
    {
      "epoch": 2.224,
      "grad_norm": 2.0692502927096633e-09,
      "learning_rate": 1.4370370370370368e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.7421875,
      "logps/chosen": -230.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 29.5,
      "rewards/rejected": -38.0,
      "step": 5560
    },
    {
      "epoch": 2.228,
      "grad_norm": 7.607808566231662e-11,
      "learning_rate": 1.4296296296296296e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -219.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 29.875,
      "rewards/rejected": -37.5,
      "step": 5570
    },
    {
      "epoch": 2.232,
      "grad_norm": 9.433361943668618e-08,
      "learning_rate": 1.4222222222222222e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.71484375,
      "logps/chosen": -254.0,
      "logps/rejected": -488.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 26.25,
      "rewards/rejected": -34.0,
      "step": 5580
    },
    {
      "epoch": 2.2359999999999998,
      "grad_norm": 5.406785248090175e-11,
      "learning_rate": 1.4148148148148147e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.796875,
      "logps/chosen": -227.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 29.75,
      "rewards/rejected": -38.25,
      "step": 5590
    },
    {
      "epoch": 2.24,
      "grad_norm": 7.059902226889724e-06,
      "learning_rate": 1.4074074074074075e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.765625,
      "logps/chosen": -211.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 26.25,
      "rewards/rejected": -34.5,
      "step": 5600
    },
    {
      "epoch": 2.2439999999999998,
      "grad_norm": 2.0576249037404712e-08,
      "learning_rate": 1.4e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.75390625,
      "logps/chosen": -242.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 28.875,
      "rewards/rejected": -36.5,
      "step": 5610
    },
    {
      "epoch": 2.248,
      "grad_norm": 5.7123975221168175e-08,
      "learning_rate": 1.3925925925925924e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.828125,
      "logps/chosen": -231.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.625,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.75,
      "step": 5620
    },
    {
      "epoch": 2.252,
      "grad_norm": 5.298548956062871e-09,
      "learning_rate": 1.385185185185185e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.78125,
      "logps/chosen": -225.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 29.875,
      "rewards/rejected": -37.5,
      "step": 5630
    },
    {
      "epoch": 2.2560000000000002,
      "grad_norm": 1.1239613745391511e-12,
      "learning_rate": 1.3777777777777778e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.77734375,
      "logps/chosen": -214.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 30.5,
      "rewards/rejected": -37.75,
      "step": 5640
    },
    {
      "epoch": 2.26,
      "grad_norm": 9.977069530596507e-07,
      "learning_rate": 1.3703703703703703e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.671875,
      "logps/chosen": -221.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.5,
      "step": 5650
    },
    {
      "epoch": 2.2640000000000002,
      "grad_norm": 4.902944670365042e-08,
      "learning_rate": 1.362962962962963e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.8046875,
      "logps/chosen": -216.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.59375,
      "rewards/margins": 27.0,
      "rewards/rejected": -34.5,
      "step": 5660
    },
    {
      "epoch": 2.268,
      "grad_norm": 6.08895668835191e-11,
      "learning_rate": 1.3555555555555557e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.88671875,
      "logps/chosen": -227.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.5,
      "step": 5670
    },
    {
      "epoch": 2.2720000000000002,
      "grad_norm": 7.289146297326861e-10,
      "learning_rate": 1.348148148148148e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.78125,
      "logps/chosen": -213.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 28.875,
      "rewards/rejected": -36.75,
      "step": 5680
    },
    {
      "epoch": 2.276,
      "grad_norm": 1.7822083961993543e-11,
      "learning_rate": 1.3407407407407405e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.80859375,
      "logps/chosen": -214.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 5690
    },
    {
      "epoch": 2.2800000000000002,
      "grad_norm": 2.957461392032744e-11,
      "learning_rate": 1.3333333333333334e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.75,
      "logps/chosen": -241.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 29.625,
      "rewards/rejected": -38.0,
      "step": 5700
    },
    {
      "epoch": 2.284,
      "grad_norm": 7.066276634873977e-10,
      "learning_rate": 1.325925925925926e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.6953125,
      "logps/chosen": -213.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.5,
      "step": 5710
    },
    {
      "epoch": 2.288,
      "grad_norm": 2.5092205755312087e-09,
      "learning_rate": 1.3185185185185185e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -215.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.5,
      "step": 5720
    },
    {
      "epoch": 2.292,
      "grad_norm": 1.9646424946750175e-06,
      "learning_rate": 1.3111111111111113e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -239.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.75,
      "step": 5730
    },
    {
      "epoch": 2.296,
      "grad_norm": 9.44136248837249e-08,
      "learning_rate": 1.3037037037037036e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.78125,
      "logps/chosen": -231.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 27.375,
      "rewards/rejected": -35.0,
      "step": 5740
    },
    {
      "epoch": 2.3,
      "grad_norm": 8.132026273930273e-08,
      "learning_rate": 1.2962962962962961e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.82421875,
      "logps/chosen": -217.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.0,
      "step": 5750
    },
    {
      "epoch": 2.304,
      "grad_norm": 7.495922175831973e-08,
      "learning_rate": 1.2888888888888887e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.75,
      "logps/chosen": -232.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.25,
      "step": 5760
    },
    {
      "epoch": 2.308,
      "grad_norm": 1.0324838508724468e-09,
      "learning_rate": 1.2814814814814815e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.77734375,
      "logps/chosen": -211.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.21875,
      "rewards/margins": 28.75,
      "rewards/rejected": -36.0,
      "step": 5770
    },
    {
      "epoch": 2.312,
      "grad_norm": 5.0809092154429055e-09,
      "learning_rate": 1.274074074074074e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -239.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 30.0,
      "rewards/rejected": -37.75,
      "step": 5780
    },
    {
      "epoch": 2.316,
      "grad_norm": 2.3291674715027587e-09,
      "learning_rate": 1.2666666666666666e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.671875,
      "logps/chosen": -229.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 31.0,
      "rewards/rejected": -38.5,
      "step": 5790
    },
    {
      "epoch": 2.32,
      "grad_norm": 1.2910390846188115e-09,
      "learning_rate": 1.2592592592592592e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.80859375,
      "logps/chosen": -260.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.75,
      "step": 5800
    },
    {
      "epoch": 2.324,
      "grad_norm": 1.9994048363472396e-07,
      "learning_rate": 1.2518518518518517e-07,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -199.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.75,
      "step": 5810
    },
    {
      "epoch": 2.328,
      "grad_norm": 4.609393074683247e-12,
      "learning_rate": 1.2444444444444443e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.72265625,
      "logps/chosen": -207.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 30.375,
      "rewards/rejected": -38.25,
      "step": 5820
    },
    {
      "epoch": 2.332,
      "grad_norm": 2.4980711240110654e-08,
      "learning_rate": 1.237037037037037e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.82421875,
      "logps/chosen": -254.0,
      "logps/rejected": -490.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.9375,
      "rewards/margins": 25.375,
      "rewards/rejected": -34.25,
      "step": 5830
    },
    {
      "epoch": 2.336,
      "grad_norm": 4.1619730711339056e-07,
      "learning_rate": 1.2296296296296297e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.796875,
      "logps/chosen": -222.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 5840
    },
    {
      "epoch": 2.34,
      "grad_norm": 6.025970813263808e-11,
      "learning_rate": 1.2222222222222222e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.75390625,
      "logps/chosen": -208.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 26.5,
      "rewards/rejected": -34.75,
      "step": 5850
    },
    {
      "epoch": 2.344,
      "grad_norm": 4.5485650529286563e-10,
      "learning_rate": 1.2148148148148148e-07,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.796875,
      "logps/chosen": -190.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.5,
      "step": 5860
    },
    {
      "epoch": 2.348,
      "grad_norm": 1.0061855358747277e-09,
      "learning_rate": 1.2074074074074073e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.82421875,
      "logps/chosen": -251.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.5,
      "step": 5870
    },
    {
      "epoch": 2.352,
      "grad_norm": 4.184573514087648e-11,
      "learning_rate": 1.2e-07,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.70703125,
      "logps/chosen": -214.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.25,
      "step": 5880
    },
    {
      "epoch": 2.356,
      "grad_norm": 1.049901542566514e-08,
      "learning_rate": 1.1925925925925924e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.71875,
      "logps/chosen": -210.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 28.875,
      "rewards/rejected": -36.75,
      "step": 5890
    },
    {
      "epoch": 2.36,
      "grad_norm": 3.219981927968928e-09,
      "learning_rate": 1.1851851851851851e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.77734375,
      "logps/chosen": -215.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.75,
      "step": 5900
    },
    {
      "epoch": 2.364,
      "grad_norm": 1.878352125784104e-07,
      "learning_rate": 1.1777777777777777e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -213.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.53125,
      "rewards/margins": 30.125,
      "rewards/rejected": -37.75,
      "step": 5910
    },
    {
      "epoch": 2.368,
      "grad_norm": 2.915356771613855e-09,
      "learning_rate": 1.1703703703703702e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.84375,
      "logps/chosen": -201.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 26.875,
      "rewards/rejected": -34.75,
      "step": 5920
    },
    {
      "epoch": 2.372,
      "grad_norm": 2.382002622145242e-11,
      "learning_rate": 1.162962962962963e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.8359375,
      "logps/chosen": -191.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.5,
      "step": 5930
    },
    {
      "epoch": 2.376,
      "grad_norm": 3.352807110850327e-08,
      "learning_rate": 1.1555555555555555e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -202.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.0,
      "step": 5940
    },
    {
      "epoch": 2.38,
      "grad_norm": 2.806284050149828e-10,
      "learning_rate": 1.148148148148148e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.828125,
      "logps/chosen": -231.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.5,
      "step": 5950
    },
    {
      "epoch": 2.384,
      "grad_norm": 5.0873360367104905e-09,
      "learning_rate": 1.1407407407407407e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.7734375,
      "logps/chosen": -228.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 29.125,
      "rewards/rejected": -36.75,
      "step": 5960
    },
    {
      "epoch": 2.388,
      "grad_norm": 8.559252623274962e-07,
      "learning_rate": 1.1333333333333332e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.90625,
      "logps/chosen": -239.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.25,
      "step": 5970
    },
    {
      "epoch": 2.392,
      "grad_norm": 2.705204294559352e-06,
      "learning_rate": 1.1259259259259258e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.83984375,
      "logps/chosen": -226.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 29.875,
      "rewards/rejected": -38.0,
      "step": 5980
    },
    {
      "epoch": 2.396,
      "grad_norm": 2.6601119984908073e-08,
      "learning_rate": 1.1185185185185185e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.83984375,
      "logps/chosen": -231.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 5990
    },
    {
      "epoch": 2.4,
      "grad_norm": 1.210521252315154e-09,
      "learning_rate": 1.111111111111111e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.6953125,
      "logps/chosen": -209.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.3125,
      "rewards/margins": 29.375,
      "rewards/rejected": -36.75,
      "step": 6000
    },
    {
      "epoch": 2.404,
      "grad_norm": 1.893014969196771e-08,
      "learning_rate": 1.1037037037037036e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -235.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 6010
    },
    {
      "epoch": 2.408,
      "grad_norm": 4.430410852502962e-12,
      "learning_rate": 1.0962962962962963e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.74609375,
      "logps/chosen": -258.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.75,
      "step": 6020
    },
    {
      "epoch": 2.412,
      "grad_norm": 4.179576739499242e-08,
      "learning_rate": 1.0888888888888888e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.84765625,
      "logps/chosen": -203.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 30.75,
      "rewards/rejected": -38.75,
      "step": 6030
    },
    {
      "epoch": 2.416,
      "grad_norm": 7.2221445251163505e-06,
      "learning_rate": 1.0814814814814814e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8046875,
      "logps/chosen": -256.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 28.75,
      "rewards/rejected": -37.25,
      "step": 6040
    },
    {
      "epoch": 2.42,
      "grad_norm": 3.159015818868911e-08,
      "learning_rate": 1.074074074074074e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.75,
      "logps/chosen": -234.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 28.375,
      "rewards/rejected": -37.0,
      "step": 6050
    },
    {
      "epoch": 2.424,
      "grad_norm": 2.33257412472934e-07,
      "learning_rate": 1.0666666666666667e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.79296875,
      "logps/chosen": -216.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.75,
      "step": 6060
    },
    {
      "epoch": 2.428,
      "grad_norm": 9.206559687217326e-09,
      "learning_rate": 1.0592592592592592e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.828125,
      "logps/chosen": -197.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 30.25,
      "rewards/rejected": -38.0,
      "step": 6070
    },
    {
      "epoch": 2.432,
      "grad_norm": 1.9949805609304513e-13,
      "learning_rate": 1.0518518518518518e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.7734375,
      "logps/chosen": -198.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.28125,
      "rewards/margins": 29.625,
      "rewards/rejected": -37.0,
      "step": 6080
    },
    {
      "epoch": 2.436,
      "grad_norm": 1.7191630471219338e-07,
      "learning_rate": 1.0444444444444445e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.6484375,
      "logps/chosen": -228.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 27.5,
      "rewards/rejected": -36.0,
      "step": 6090
    },
    {
      "epoch": 2.44,
      "grad_norm": 3.6415781163375375e-10,
      "learning_rate": 1.0370370370370369e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.67578125,
      "logps/chosen": -206.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 29.0,
      "rewards/rejected": -37.25,
      "step": 6100
    },
    {
      "epoch": 2.444,
      "grad_norm": 5.549506596113482e-10,
      "learning_rate": 1.0296296296296296e-07,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.83984375,
      "logps/chosen": -239.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.25,
      "step": 6110
    },
    {
      "epoch": 2.448,
      "grad_norm": 3.1639992583396143e-09,
      "learning_rate": 1.0222222222222223e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.75,
      "logps/chosen": -228.0,
      "logps/rejected": -544.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 31.25,
      "rewards/rejected": -39.25,
      "step": 6120
    },
    {
      "epoch": 2.452,
      "grad_norm": 9.211380803826552e-11,
      "learning_rate": 1.0148148148148147e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.7109375,
      "logps/chosen": -213.0,
      "logps/rejected": -544.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.75,
      "step": 6130
    },
    {
      "epoch": 2.456,
      "grad_norm": 3.3393187536626437e-06,
      "learning_rate": 1.0074074074074074e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.73828125,
      "logps/chosen": -256.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 31.0,
      "rewards/rejected": -39.0,
      "step": 6140
    },
    {
      "epoch": 2.46,
      "grad_norm": 6.29431141952154e-08,
      "learning_rate": 1e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.828125,
      "logps/chosen": -241.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 30.0,
      "rewards/rejected": -38.0,
      "step": 6150
    },
    {
      "epoch": 2.464,
      "grad_norm": 4.554866813024335e-09,
      "learning_rate": 9.925925925925925e-08,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.69921875,
      "logps/chosen": -238.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.5,
      "step": 6160
    },
    {
      "epoch": 2.468,
      "grad_norm": 8.248011946133724e-12,
      "learning_rate": 9.851851851851852e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -193.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.21875,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.0,
      "step": 6170
    },
    {
      "epoch": 2.472,
      "grad_norm": 3.2455496958983797e-06,
      "learning_rate": 9.777777777777778e-08,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.75,
      "logps/chosen": -210.0,
      "logps/rejected": -494.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.65625,
      "rewards/margins": 27.125,
      "rewards/rejected": -34.75,
      "step": 6180
    },
    {
      "epoch": 2.476,
      "grad_norm": 1.3590181889201553e-07,
      "learning_rate": 9.703703703703703e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.6484375,
      "logps/chosen": -209.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.75,
      "step": 6190
    },
    {
      "epoch": 2.48,
      "grad_norm": 7.055848287612723e-11,
      "learning_rate": 9.629629629629629e-08,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -234.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 27.625,
      "rewards/rejected": -36.0,
      "step": 6200
    },
    {
      "epoch": 2.484,
      "grad_norm": 1.0141126992524514e-06,
      "learning_rate": 9.555555555555556e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.76171875,
      "logps/chosen": -218.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 28.875,
      "rewards/rejected": -36.75,
      "step": 6210
    },
    {
      "epoch": 2.488,
      "grad_norm": 7.948322154501513e-11,
      "learning_rate": 9.481481481481481e-08,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.765625,
      "logps/chosen": -195.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 31.125,
      "rewards/rejected": -39.0,
      "step": 6220
    },
    {
      "epoch": 2.492,
      "grad_norm": 4.346092990146117e-07,
      "learning_rate": 9.407407407407407e-08,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.75390625,
      "logps/chosen": -234.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 27.5,
      "rewards/rejected": -36.0,
      "step": 6230
    },
    {
      "epoch": 2.496,
      "grad_norm": 8.006054890864703e-11,
      "learning_rate": 9.333333333333334e-08,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.70703125,
      "logps/chosen": -206.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 31.125,
      "rewards/rejected": -39.0,
      "step": 6240
    },
    {
      "epoch": 2.5,
      "grad_norm": 7.393159289615492e-10,
      "learning_rate": 9.259259259259258e-08,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -227.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.75,
      "step": 6250
    },
    {
      "epoch": 2.504,
      "grad_norm": 1.3393447164805253e-10,
      "learning_rate": 9.185185185185185e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -198.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 30.875,
      "rewards/rejected": -38.75,
      "step": 6260
    },
    {
      "epoch": 2.508,
      "grad_norm": 1.0753328257031809e-10,
      "learning_rate": 9.111111111111112e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.7265625,
      "logps/chosen": -223.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.75,
      "step": 6270
    },
    {
      "epoch": 2.512,
      "grad_norm": 1.4447774171210512e-09,
      "learning_rate": 9.037037037037036e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.890625,
      "logps/chosen": -224.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.25,
      "step": 6280
    },
    {
      "epoch": 2.516,
      "grad_norm": 7.399612334358668e-09,
      "learning_rate": 8.962962962962963e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.84765625,
      "logps/chosen": -213.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.0,
      "step": 6290
    },
    {
      "epoch": 2.52,
      "grad_norm": 5.980651106172293e-08,
      "learning_rate": 8.888888888888888e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -217.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 27.625,
      "rewards/rejected": -35.25,
      "step": 6300
    },
    {
      "epoch": 2.524,
      "grad_norm": 6.27457708771239e-10,
      "learning_rate": 8.814814814814814e-08,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.7265625,
      "logps/chosen": -256.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 29.125,
      "rewards/rejected": -36.75,
      "step": 6310
    },
    {
      "epoch": 2.528,
      "grad_norm": 1.8216306821705998e-06,
      "learning_rate": 8.74074074074074e-08,
      "logits/chosen": -1.0,
      "logits/rejected": -0.8125,
      "logps/chosen": -206.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 29.125,
      "rewards/rejected": -37.0,
      "step": 6320
    },
    {
      "epoch": 2.532,
      "grad_norm": 3.8797058840525976e-09,
      "learning_rate": 8.666666666666666e-08,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.8515625,
      "logps/chosen": -211.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.25,
      "step": 6330
    },
    {
      "epoch": 2.536,
      "grad_norm": 1.596508083424143e-10,
      "learning_rate": 8.592592592592592e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.78515625,
      "logps/chosen": -208.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 29.875,
      "rewards/rejected": -37.75,
      "step": 6340
    },
    {
      "epoch": 2.54,
      "grad_norm": 3.7690447509553163e-07,
      "learning_rate": 8.518518518518517e-08,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.859375,
      "logps/chosen": -217.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 26.625,
      "rewards/rejected": -34.75,
      "step": 6350
    },
    {
      "epoch": 2.544,
      "grad_norm": 6.358938450901739e-06,
      "learning_rate": 8.444444444444444e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.74609375,
      "logps/chosen": -235.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 30.0,
      "rewards/rejected": -38.0,
      "step": 6360
    },
    {
      "epoch": 2.548,
      "grad_norm": 7.908426197225754e-09,
      "learning_rate": 8.37037037037037e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.79296875,
      "logps/chosen": -235.0,
      "logps/rejected": -540.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.75,
      "step": 6370
    },
    {
      "epoch": 2.552,
      "grad_norm": 1.515876922689504e-09,
      "learning_rate": 8.296296296296295e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.703125,
      "logps/chosen": -211.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5625,
      "rewards/margins": 30.0,
      "rewards/rejected": -37.5,
      "step": 6380
    },
    {
      "epoch": 2.556,
      "grad_norm": 5.254707470912444e-10,
      "learning_rate": 8.222222222222222e-08,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.91015625,
      "logps/chosen": -224.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 27.625,
      "rewards/rejected": -36.0,
      "step": 6390
    },
    {
      "epoch": 2.56,
      "grad_norm": 4.460133344252526e-10,
      "learning_rate": 8.148148148148149e-08,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.828125,
      "logps/chosen": -250.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 29.625,
      "rewards/rejected": -38.0,
      "step": 6400
    },
    {
      "epoch": 2.564,
      "grad_norm": 8.623916169974308e-11,
      "learning_rate": 8.074074074074073e-08,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -234.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.65625,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.0,
      "step": 6410
    },
    {
      "epoch": 2.568,
      "grad_norm": 2.9634463618825077e-11,
      "learning_rate": 8e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.76171875,
      "logps/chosen": -209.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.25,
      "step": 6420
    },
    {
      "epoch": 2.572,
      "grad_norm": 2.5369927984398126e-11,
      "learning_rate": 7.925925925925926e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.76953125,
      "logps/chosen": -230.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 30.375,
      "rewards/rejected": -38.5,
      "step": 6430
    },
    {
      "epoch": 2.576,
      "grad_norm": 2.411572287265087e-10,
      "learning_rate": 7.851851851851851e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -207.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 6440
    },
    {
      "epoch": 2.58,
      "grad_norm": 6.013785939121597e-09,
      "learning_rate": 7.777777777777778e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.7578125,
      "logps/chosen": -206.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.75,
      "step": 6450
    },
    {
      "epoch": 2.584,
      "grad_norm": 5.884571835863563e-09,
      "learning_rate": 7.703703703703704e-08,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -229.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.0,
      "rewards/margins": 28.25,
      "rewards/rejected": -37.25,
      "step": 6460
    },
    {
      "epoch": 2.588,
      "grad_norm": 1.3874579984686862e-10,
      "learning_rate": 7.629629629629629e-08,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.796875,
      "logps/chosen": -264.0,
      "logps/rejected": -540.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 31.0,
      "rewards/rejected": -39.25,
      "step": 6470
    },
    {
      "epoch": 2.592,
      "grad_norm": 0.00011209436755123213,
      "learning_rate": 7.555555555555555e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.7421875,
      "logps/chosen": -212.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.25,
      "step": 6480
    },
    {
      "epoch": 2.596,
      "grad_norm": 2.613731228731614e-09,
      "learning_rate": 7.481481481481482e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -220.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.75,
      "rewards/margins": 26.625,
      "rewards/rejected": -35.5,
      "step": 6490
    },
    {
      "epoch": 2.6,
      "grad_norm": 7.077837870064484e-08,
      "learning_rate": 7.407407407407407e-08,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -242.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 27.625,
      "rewards/rejected": -36.0,
      "step": 6500
    },
    {
      "epoch": 2.604,
      "grad_norm": 2.0133725853325548e-12,
      "learning_rate": 7.333333333333333e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.66796875,
      "logps/chosen": -228.0,
      "logps/rejected": -548.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 31.375,
      "rewards/rejected": -40.0,
      "step": 6510
    },
    {
      "epoch": 2.608,
      "grad_norm": 3.836428021158305e-06,
      "learning_rate": 7.25925925925926e-08,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.7734375,
      "logps/chosen": -240.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.8125,
      "rewards/margins": 27.25,
      "rewards/rejected": -36.0,
      "step": 6520
    },
    {
      "epoch": 2.612,
      "grad_norm": 3.6560811429235145e-10,
      "learning_rate": 7.185185185185184e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.8515625,
      "logps/chosen": -214.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 29.875,
      "rewards/rejected": -38.0,
      "step": 6530
    },
    {
      "epoch": 2.616,
      "grad_norm": 1.3503008181173223e-10,
      "learning_rate": 7.111111111111111e-08,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.828125,
      "logps/chosen": -219.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.75,
      "step": 6540
    },
    {
      "epoch": 2.62,
      "grad_norm": 2.3475728969930174e-09,
      "learning_rate": 7.037037037037038e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.8671875,
      "logps/chosen": -212.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 29.5,
      "rewards/rejected": -38.0,
      "step": 6550
    },
    {
      "epoch": 2.624,
      "grad_norm": 2.449664699680499e-09,
      "learning_rate": 6.962962962962962e-08,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.79296875,
      "logps/chosen": -250.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 28.125,
      "rewards/rejected": -36.5,
      "step": 6560
    },
    {
      "epoch": 2.628,
      "grad_norm": 2.861628279935079e-10,
      "learning_rate": 6.888888888888889e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.74609375,
      "logps/chosen": -207.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 31.0,
      "rewards/rejected": -38.75,
      "step": 6570
    },
    {
      "epoch": 2.632,
      "grad_norm": 6.561739386681361e-07,
      "learning_rate": 6.814814814814814e-08,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -237.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.75,
      "step": 6580
    },
    {
      "epoch": 2.636,
      "grad_norm": 2.0640230875773484e-10,
      "learning_rate": 6.74074074074074e-08,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.6875,
      "logps/chosen": -222.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.75,
      "step": 6590
    },
    {
      "epoch": 2.64,
      "grad_norm": 3.861287018587555e-10,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.7578125,
      "logps/chosen": -223.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 30.125,
      "rewards/rejected": -38.75,
      "step": 6600
    },
    {
      "epoch": 2.644,
      "grad_norm": 5.168556723294027e-07,
      "learning_rate": 6.592592592592592e-08,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.83203125,
      "logps/chosen": -210.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 30.75,
      "rewards/rejected": -39.0,
      "step": 6610
    },
    {
      "epoch": 2.648,
      "grad_norm": 1.825127997020314e-06,
      "learning_rate": 6.518518518518518e-08,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.78125,
      "logps/chosen": -234.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.0,
      "step": 6620
    },
    {
      "epoch": 2.652,
      "grad_norm": 4.120443205351969e-11,
      "learning_rate": 6.444444444444443e-08,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.87109375,
      "logps/chosen": -254.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 27.5,
      "rewards/rejected": -35.75,
      "step": 6630
    },
    {
      "epoch": 2.656,
      "grad_norm": 2.440034361339458e-10,
      "learning_rate": 6.37037037037037e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.84375,
      "logps/chosen": -199.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.46875,
      "rewards/margins": 29.25,
      "rewards/rejected": -36.75,
      "step": 6640
    },
    {
      "epoch": 2.66,
      "grad_norm": 1.0560250389604556e-09,
      "learning_rate": 6.296296296296296e-08,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.83203125,
      "logps/chosen": -228.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 28.625,
      "rewards/rejected": -36.75,
      "step": 6650
    },
    {
      "epoch": 2.664,
      "grad_norm": 7.746582242264283e-09,
      "learning_rate": 6.222222222222221e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.875,
      "logps/chosen": -216.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 26.75,
      "rewards/rejected": -35.0,
      "step": 6660
    },
    {
      "epoch": 2.668,
      "grad_norm": 3.178770019868275e-06,
      "learning_rate": 6.148148148148148e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.890625,
      "logps/chosen": -236.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 27.75,
      "rewards/rejected": -35.75,
      "step": 6670
    },
    {
      "epoch": 2.672,
      "grad_norm": 2.706278601316404e-08,
      "learning_rate": 6.074074074074074e-08,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.7578125,
      "logps/chosen": -208.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.0,
      "step": 6680
    },
    {
      "epoch": 2.676,
      "grad_norm": 1.1896209699363806e-09,
      "learning_rate": 6e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.75390625,
      "logps/chosen": -222.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.25,
      "step": 6690
    },
    {
      "epoch": 2.68,
      "grad_norm": 8.704054764014498e-08,
      "learning_rate": 5.925925925925926e-08,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.80078125,
      "logps/chosen": -221.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 27.875,
      "rewards/rejected": -35.5,
      "step": 6700
    },
    {
      "epoch": 2.684,
      "grad_norm": 4.3490895112410745e-09,
      "learning_rate": 5.851851851851851e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -213.0,
      "logps/rejected": -482.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 26.125,
      "rewards/rejected": -34.5,
      "step": 6710
    },
    {
      "epoch": 2.6879999999999997,
      "grad_norm": 6.943589557018027e-08,
      "learning_rate": 5.7777777777777775e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -250.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 29.125,
      "rewards/rejected": -37.0,
      "step": 6720
    },
    {
      "epoch": 2.692,
      "grad_norm": 5.774342751460791e-09,
      "learning_rate": 5.703703703703704e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.58203125,
      "logps/chosen": -220.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.25,
      "step": 6730
    },
    {
      "epoch": 2.6959999999999997,
      "grad_norm": 6.872000912212114e-10,
      "learning_rate": 5.629629629629629e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.890625,
      "logps/chosen": -218.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.75,
      "step": 6740
    },
    {
      "epoch": 2.7,
      "grad_norm": 1.675618295805668e-06,
      "learning_rate": 5.555555555555555e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.90234375,
      "logps/chosen": -243.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.5,
      "step": 6750
    },
    {
      "epoch": 2.7039999999999997,
      "grad_norm": 1.7068744148995253e-08,
      "learning_rate": 5.481481481481482e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.69921875,
      "logps/chosen": -224.0,
      "logps/rejected": -506.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.65625,
      "rewards/margins": 27.5,
      "rewards/rejected": -35.25,
      "step": 6760
    },
    {
      "epoch": 2.708,
      "grad_norm": 5.059231509317424e-06,
      "learning_rate": 5.407407407407407e-08,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.828125,
      "logps/chosen": -236.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 29.0,
      "rewards/rejected": -37.0,
      "step": 6770
    },
    {
      "epoch": 2.7119999999999997,
      "grad_norm": 4.548974075934989e-07,
      "learning_rate": 5.3333333333333334e-08,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.828125,
      "logps/chosen": -195.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.5,
      "step": 6780
    },
    {
      "epoch": 2.716,
      "grad_norm": 0.0004615434671985879,
      "learning_rate": 5.259259259259259e-08,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.7890625,
      "logps/chosen": -222.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 26.75,
      "rewards/rejected": -35.0,
      "step": 6790
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 8.026114078351694e-11,
      "learning_rate": 5.1851851851851846e-08,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.828125,
      "logps/chosen": -234.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 29.125,
      "rewards/rejected": -37.5,
      "step": 6800
    },
    {
      "epoch": 2.724,
      "grad_norm": 1.4137490370007526e-09,
      "learning_rate": 5.1111111111111114e-08,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.640625,
      "logps/chosen": -227.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.1875,
      "rewards/margins": 28.75,
      "rewards/rejected": -38.0,
      "step": 6810
    },
    {
      "epoch": 2.7279999999999998,
      "grad_norm": 4.501403205604375e-06,
      "learning_rate": 5.037037037037037e-08,
      "logits/chosen": -1.0,
      "logits/rejected": -0.6875,
      "logps/chosen": -211.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 30.125,
      "rewards/rejected": -38.5,
      "step": 6820
    },
    {
      "epoch": 2.732,
      "grad_norm": 4.310352859960732e-08,
      "learning_rate": 4.9629629629629626e-08,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8203125,
      "logps/chosen": -245.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 30.0,
      "rewards/rejected": -37.75,
      "step": 6830
    },
    {
      "epoch": 2.7359999999999998,
      "grad_norm": 1.4858651899264625e-10,
      "learning_rate": 4.888888888888889e-08,
      "logits/chosen": -1.0,
      "logits/rejected": -0.7578125,
      "logps/chosen": -190.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 30.125,
      "rewards/rejected": -38.25,
      "step": 6840
    },
    {
      "epoch": 2.74,
      "grad_norm": 2.0053623097083705e-06,
      "learning_rate": 4.814814814814814e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.76953125,
      "logps/chosen": -229.0,
      "logps/rejected": -540.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.6875,
      "rewards/margins": 31.0,
      "rewards/rejected": -39.75,
      "step": 6850
    },
    {
      "epoch": 2.7439999999999998,
      "grad_norm": 5.949164639471148e-06,
      "learning_rate": 4.7407407407407405e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.734375,
      "logps/chosen": -204.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 30.5,
      "rewards/rejected": -38.5,
      "step": 6860
    },
    {
      "epoch": 2.748,
      "grad_norm": 5.689053376148312e-07,
      "learning_rate": 4.666666666666667e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.76171875,
      "logps/chosen": -215.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 29.125,
      "rewards/rejected": -37.25,
      "step": 6870
    },
    {
      "epoch": 2.752,
      "grad_norm": 9.115783182217892e-11,
      "learning_rate": 4.592592592592592e-08,
      "logits/chosen": -1.0,
      "logits/rejected": -0.78125,
      "logps/chosen": -194.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.71875,
      "rewards/margins": 30.0,
      "rewards/rejected": -37.75,
      "step": 6880
    },
    {
      "epoch": 2.7560000000000002,
      "grad_norm": 3.6307924279651914e-09,
      "learning_rate": 4.518518518518518e-08,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -230.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.75,
      "step": 6890
    },
    {
      "epoch": 2.76,
      "grad_norm": 7.215502010583728e-09,
      "learning_rate": 4.444444444444444e-08,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.77734375,
      "logps/chosen": -239.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 27.125,
      "rewards/rejected": -35.25,
      "step": 6900
    },
    {
      "epoch": 2.7640000000000002,
      "grad_norm": 4.82315718679611e-10,
      "learning_rate": 4.37037037037037e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -236.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 28.5,
      "rewards/rejected": -37.0,
      "step": 6910
    },
    {
      "epoch": 2.768,
      "grad_norm": 8.103018162842769e-10,
      "learning_rate": 4.296296296296296e-08,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.76953125,
      "logps/chosen": -222.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.25,
      "step": 6920
    },
    {
      "epoch": 2.7720000000000002,
      "grad_norm": 5.490071929748478e-12,
      "learning_rate": 4.222222222222222e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.765625,
      "logps/chosen": -231.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 6930
    },
    {
      "epoch": 2.776,
      "grad_norm": 7.766052514119313e-07,
      "learning_rate": 4.1481481481481476e-08,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -214.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.75,
      "rewards/rejected": -37.0,
      "step": 6940
    },
    {
      "epoch": 2.7800000000000002,
      "grad_norm": 1.961100802743893e-09,
      "learning_rate": 4.0740740740740745e-08,
      "logits/chosen": -0.90234375,
      "logits/rejected": -0.6640625,
      "logps/chosen": -185.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 30.25,
      "rewards/rejected": -38.0,
      "step": 6950
    },
    {
      "epoch": 2.784,
      "grad_norm": 3.139461144102241e-09,
      "learning_rate": 4e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.6640625,
      "logps/chosen": -227.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.5,
      "step": 6960
    },
    {
      "epoch": 2.7880000000000003,
      "grad_norm": 9.720463362924025e-07,
      "learning_rate": 3.9259259259259256e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.765625,
      "logps/chosen": -210.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 30.5,
      "rewards/rejected": -38.5,
      "step": 6970
    },
    {
      "epoch": 2.792,
      "grad_norm": 5.017944706868231e-06,
      "learning_rate": 3.851851851851852e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.76171875,
      "logps/chosen": -188.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.5,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.5,
      "step": 6980
    },
    {
      "epoch": 2.7960000000000003,
      "grad_norm": 2.646281182885989e-10,
      "learning_rate": 3.7777777777777774e-08,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.828125,
      "logps/chosen": -229.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.5,
      "step": 6990
    },
    {
      "epoch": 2.8,
      "grad_norm": 6.011976002239977e-09,
      "learning_rate": 3.7037037037037036e-08,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.796875,
      "logps/chosen": -207.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.75,
      "step": 7000
    },
    {
      "epoch": 2.8040000000000003,
      "grad_norm": 2.0330277438180265e-08,
      "learning_rate": 3.62962962962963e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.7734375,
      "logps/chosen": -217.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.9375,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 7010
    },
    {
      "epoch": 2.808,
      "grad_norm": 5.607207324030893e-09,
      "learning_rate": 3.5555555555555554e-08,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -206.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.5,
      "step": 7020
    },
    {
      "epoch": 2.8120000000000003,
      "grad_norm": 1.742828869429844e-10,
      "learning_rate": 3.481481481481481e-08,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.859375,
      "logps/chosen": -201.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.78125,
      "rewards/margins": 30.5,
      "rewards/rejected": -38.25,
      "step": 7030
    },
    {
      "epoch": 2.816,
      "grad_norm": 1.1132251619529313e-08,
      "learning_rate": 3.407407407407407e-08,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.84765625,
      "logps/chosen": -248.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 27.25,
      "rewards/rejected": -35.75,
      "step": 7040
    },
    {
      "epoch": 2.82,
      "grad_norm": 4.295876099929809e-08,
      "learning_rate": 3.3333333333333334e-08,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.5859375,
      "logps/chosen": -196.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.28125,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.25,
      "step": 7050
    },
    {
      "epoch": 2.824,
      "grad_norm": 5.51780028557814e-05,
      "learning_rate": 3.259259259259259e-08,
      "logits/chosen": -1.0,
      "logits/rejected": -0.7734375,
      "logps/chosen": -208.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.40625,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.5,
      "step": 7060
    },
    {
      "epoch": 2.828,
      "grad_norm": 6.894207915305314e-08,
      "learning_rate": 3.185185185185185e-08,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.76171875,
      "logps/chosen": -237.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.25,
      "step": 7070
    },
    {
      "epoch": 2.832,
      "grad_norm": 1.8195617166562167e-08,
      "learning_rate": 3.111111111111111e-08,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.6171875,
      "logps/chosen": -206.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.75,
      "step": 7080
    },
    {
      "epoch": 2.836,
      "grad_norm": 3.053034479547929e-09,
      "learning_rate": 3.037037037037037e-08,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -226.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.5,
      "step": 7090
    },
    {
      "epoch": 2.84,
      "grad_norm": 2.4952153957513757e-10,
      "learning_rate": 2.962962962962963e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.66015625,
      "logps/chosen": -225.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 30.875,
      "rewards/rejected": -38.75,
      "step": 7100
    },
    {
      "epoch": 2.844,
      "grad_norm": 5.376284038950498e-13,
      "learning_rate": 2.8888888888888887e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.7578125,
      "logps/chosen": -202.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 29.0,
      "rewards/rejected": -36.75,
      "step": 7110
    },
    {
      "epoch": 2.848,
      "grad_norm": 6.255099339832569e-07,
      "learning_rate": 2.8148148148148146e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -212.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.75,
      "step": 7120
    },
    {
      "epoch": 2.852,
      "grad_norm": 1.1050200549212555e-08,
      "learning_rate": 2.740740740740741e-08,
      "logits/chosen": -0.78515625,
      "logits/rejected": -0.87890625,
      "logps/chosen": -178.0,
      "logps/rejected": -510.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.5,
      "step": 7130
    },
    {
      "epoch": 2.856,
      "grad_norm": 3.455465055552877e-10,
      "learning_rate": 2.6666666666666667e-08,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.77734375,
      "logps/chosen": -226.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 29.125,
      "rewards/rejected": -37.5,
      "step": 7140
    },
    {
      "epoch": 2.86,
      "grad_norm": 4.359713011676068e-06,
      "learning_rate": 2.5925925925925923e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.89453125,
      "logps/chosen": -226.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5,
      "rewards/margins": 26.75,
      "rewards/rejected": -35.25,
      "step": 7150
    },
    {
      "epoch": 2.864,
      "grad_norm": 5.064654051149542e-10,
      "learning_rate": 2.5185185185185185e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.6171875,
      "logps/chosen": -210.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 30.125,
      "rewards/rejected": -38.25,
      "step": 7160
    },
    {
      "epoch": 2.868,
      "grad_norm": 1.0234110847314968e-06,
      "learning_rate": 2.4444444444444444e-08,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.7109375,
      "logps/chosen": -236.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.75,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.75,
      "step": 7170
    },
    {
      "epoch": 2.872,
      "grad_norm": 2.9964405688359367e-08,
      "learning_rate": 2.3703703703703703e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.69140625,
      "logps/chosen": -241.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 30.0,
      "rewards/rejected": -38.5,
      "step": 7180
    },
    {
      "epoch": 2.876,
      "grad_norm": 1.5070722858727233e-07,
      "learning_rate": 2.296296296296296e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.84765625,
      "logps/chosen": -225.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.46875,
      "rewards/margins": 29.75,
      "rewards/rejected": -37.25,
      "step": 7190
    },
    {
      "epoch": 2.88,
      "grad_norm": 3.635446313580513e-06,
      "learning_rate": 2.222222222222222e-08,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.75390625,
      "logps/chosen": -216.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 28.875,
      "rewards/rejected": -37.25,
      "step": 7200
    },
    {
      "epoch": 2.884,
      "grad_norm": 2.431348020987113e-09,
      "learning_rate": 2.148148148148148e-08,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.8203125,
      "logps/chosen": -197.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.125,
      "rewards/margins": 29.375,
      "rewards/rejected": -37.5,
      "step": 7210
    },
    {
      "epoch": 2.888,
      "grad_norm": 1.8872392434149956e-12,
      "learning_rate": 2.0740740740740738e-08,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.7109375,
      "logps/chosen": -216.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.6875,
      "rewards/margins": 30.625,
      "rewards/rejected": -38.25,
      "step": 7220
    },
    {
      "epoch": 2.892,
      "grad_norm": 3.869894155383819e-09,
      "learning_rate": 2e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.84375,
      "logps/chosen": -213.0,
      "logps/rejected": -508.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 28.5,
      "rewards/rejected": -36.25,
      "step": 7230
    },
    {
      "epoch": 2.896,
      "grad_norm": 6.253707563166895e-06,
      "learning_rate": 1.925925925925926e-08,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.81640625,
      "logps/chosen": -227.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 27.5,
      "rewards/rejected": -35.75,
      "step": 7240
    },
    {
      "epoch": 2.9,
      "grad_norm": 1.6962696925385857e-10,
      "learning_rate": 1.8518518518518518e-08,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.78125,
      "logps/chosen": -237.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.25,
      "step": 7250
    },
    {
      "epoch": 2.904,
      "grad_norm": 1.03798556591574e-08,
      "learning_rate": 1.7777777777777777e-08,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.72265625,
      "logps/chosen": -223.0,
      "logps/rejected": -548.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.1875,
      "rewards/margins": 31.25,
      "rewards/rejected": -39.5,
      "step": 7260
    },
    {
      "epoch": 2.908,
      "grad_norm": 2.228554035853569e-06,
      "learning_rate": 1.7037037037037036e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.8125,
      "logps/chosen": -231.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.625,
      "rewards/margins": 28.25,
      "rewards/rejected": -36.75,
      "step": 7270
    },
    {
      "epoch": 2.912,
      "grad_norm": 4.16458189565665e-12,
      "learning_rate": 1.6296296296296295e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.734375,
      "logps/chosen": -214.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 30.5,
      "rewards/rejected": -38.5,
      "step": 7280
    },
    {
      "epoch": 2.916,
      "grad_norm": 3.6523446233360814e-10,
      "learning_rate": 1.5555555555555554e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.765625,
      "logps/chosen": -219.0,
      "logps/rejected": -528.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 29.5,
      "rewards/rejected": -38.0,
      "step": 7290
    },
    {
      "epoch": 2.92,
      "grad_norm": 1.6092626669810766e-06,
      "learning_rate": 1.4814814814814814e-08,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.75390625,
      "logps/chosen": -215.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 29.0,
      "rewards/rejected": -37.25,
      "step": 7300
    },
    {
      "epoch": 2.924,
      "grad_norm": 5.271251421799271e-07,
      "learning_rate": 1.4074074074074073e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.81640625,
      "logps/chosen": -242.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0625,
      "rewards/margins": 27.5,
      "rewards/rejected": -35.5,
      "step": 7310
    },
    {
      "epoch": 2.928,
      "grad_norm": 1.0688903561325734e-06,
      "learning_rate": 1.3333333333333334e-08,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.6796875,
      "logps/chosen": -232.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 27.125,
      "rewards/rejected": -35.5,
      "step": 7320
    },
    {
      "epoch": 2.932,
      "grad_norm": 9.464517427973233e-10,
      "learning_rate": 1.2592592592592592e-08,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.765625,
      "logps/chosen": -191.0,
      "logps/rejected": -498.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.8125,
      "rewards/margins": 28.375,
      "rewards/rejected": -36.25,
      "step": 7330
    },
    {
      "epoch": 2.936,
      "grad_norm": 5.086862082441263e-07,
      "learning_rate": 1.1851851851851851e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.77734375,
      "logps/chosen": -205.0,
      "logps/rejected": -502.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.96875,
      "rewards/margins": 28.0,
      "rewards/rejected": -36.0,
      "step": 7340
    },
    {
      "epoch": 2.94,
      "grad_norm": 3.5931107191238397e-10,
      "learning_rate": 1.111111111111111e-08,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.80859375,
      "logps/chosen": -262.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 29.625,
      "rewards/rejected": -37.5,
      "step": 7350
    },
    {
      "epoch": 2.944,
      "grad_norm": 4.469950194237821e-07,
      "learning_rate": 1.0370370370370369e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.6796875,
      "logps/chosen": -241.0,
      "logps/rejected": -492.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 26.125,
      "rewards/rejected": -34.75,
      "step": 7360
    },
    {
      "epoch": 2.948,
      "grad_norm": 1.170084266450381e-09,
      "learning_rate": 9.62962962962963e-09,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.80859375,
      "logps/chosen": -229.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.25,
      "rewards/margins": 29.125,
      "rewards/rejected": -37.5,
      "step": 7370
    },
    {
      "epoch": 2.952,
      "grad_norm": 1.2125297301083517e-08,
      "learning_rate": 8.888888888888889e-09,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -222.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.90625,
      "rewards/margins": 29.25,
      "rewards/rejected": -37.0,
      "step": 7380
    },
    {
      "epoch": 2.956,
      "grad_norm": 1.318450671294693e-09,
      "learning_rate": 8.148148148148147e-09,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.8046875,
      "logps/chosen": -235.0,
      "logps/rejected": -512.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.375,
      "rewards/margins": 28.5,
      "rewards/rejected": -37.0,
      "step": 7390
    },
    {
      "epoch": 2.96,
      "grad_norm": 5.209022589577651e-08,
      "learning_rate": 7.407407407407407e-09,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.85546875,
      "logps/chosen": -198.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.5625,
      "rewards/margins": 27.875,
      "rewards/rejected": -36.5,
      "step": 7400
    },
    {
      "epoch": 2.964,
      "grad_norm": 2.7716483724875325e-08,
      "learning_rate": 6.666666666666667e-09,
      "logits/chosen": -1.125,
      "logits/rejected": -0.87890625,
      "logps/chosen": -262.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -9.25,
      "rewards/margins": 28.375,
      "rewards/rejected": -37.5,
      "step": 7410
    },
    {
      "epoch": 2.968,
      "grad_norm": 3.440977553894175e-11,
      "learning_rate": 5.925925925925926e-09,
      "logits/chosen": -1.125,
      "logits/rejected": -0.8203125,
      "logps/chosen": -214.0,
      "logps/rejected": -500.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.75,
      "rewards/margins": 27.0,
      "rewards/rejected": -35.0,
      "step": 7420
    },
    {
      "epoch": 2.972,
      "grad_norm": 3.796144785983056e-09,
      "learning_rate": 5.1851851851851846e-09,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.87890625,
      "logps/chosen": -234.0,
      "logps/rejected": -516.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.84375,
      "rewards/margins": 29.5,
      "rewards/rejected": -37.25,
      "step": 7430
    },
    {
      "epoch": 2.976,
      "grad_norm": 2.995813860878697e-08,
      "learning_rate": 4.444444444444444e-09,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.80859375,
      "logps/chosen": -201.0,
      "logps/rejected": -524.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.0,
      "rewards/margins": 30.25,
      "rewards/rejected": -38.25,
      "step": 7440
    },
    {
      "epoch": 2.98,
      "grad_norm": 2.0264744081122967e-08,
      "learning_rate": 3.7037037037037036e-09,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.875,
      "logps/chosen": -250.0,
      "logps/rejected": -536.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 29.75,
      "rewards/rejected": -38.25,
      "step": 7450
    },
    {
      "epoch": 2.984,
      "grad_norm": 5.325174564691763e-09,
      "learning_rate": 2.962962962962963e-09,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -252.0,
      "logps/rejected": -496.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.8125,
      "rewards/margins": 26.0,
      "rewards/rejected": -34.75,
      "step": 7460
    },
    {
      "epoch": 2.988,
      "grad_norm": 3.4095680292148088e-12,
      "learning_rate": 2.222222222222222e-09,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.83203125,
      "logps/chosen": -229.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.4375,
      "rewards/margins": 30.375,
      "rewards/rejected": -38.75,
      "step": 7470
    },
    {
      "epoch": 2.992,
      "grad_norm": 5.860359686437275e-07,
      "learning_rate": 1.4814814814814814e-09,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.76171875,
      "logps/chosen": -222.0,
      "logps/rejected": -532.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.875,
      "rewards/margins": 30.375,
      "rewards/rejected": -38.25,
      "step": 7480
    },
    {
      "epoch": 2.996,
      "grad_norm": 8.088706662665376e-09,
      "learning_rate": 7.407407407407407e-10,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.76171875,
      "logps/chosen": -220.0,
      "logps/rejected": -504.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.3125,
      "rewards/margins": 27.375,
      "rewards/rejected": -35.75,
      "step": 7490
    },
    {
      "epoch": 3.0,
      "grad_norm": 3.6419933094672554e-10,
      "learning_rate": 0.0,
      "logits/chosen": -1.125,
      "logits/rejected": -0.73828125,
      "logps/chosen": -231.0,
      "logps/rejected": -520.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.0625,
      "rewards/margins": 30.875,
      "rewards/rejected": -38.0,
      "step": 7500
    },
    {
      "epoch": 3.0,
      "eval_logits/chosen": -1.1171875,
      "eval_logits/rejected": -0.73828125,
      "eval_logps/chosen": -225.0,
      "eval_logps/rejected": -520.0,
      "eval_loss": 1.3488392625049528e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -8.125,
      "eval_rewards/margins": 29.375,
      "eval_rewards/rejected": -37.5,
      "eval_runtime": 55.4147,
      "eval_samples_per_second": 3.609,
      "eval_steps_per_second": 0.902,
      "step": 7500
    },
    {
      "epoch": 3.0,
      "step": 7500,
      "total_flos": 0.0,
      "train_loss": 0.018431058258183003,
      "train_runtime": 15031.6783,
      "train_samples_per_second": 1.996,
      "train_steps_per_second": 0.499
    }
  ],
  "logging_steps": 10,
  "max_steps": 7500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}